Buckets:
| import"../chunks/DsnmJJEf.js";import{i as $a,h as Ka,C as eo,H as s,D as t,E as to,s as ao,a as p}from"../chunks/BnX7_rEC.js";import{p as oo,o as ro,s as e,f as m,a as d,b as no,c as a,d as _,n as r,r as o}from"../chunks/Cq6T2TQD.js";import{T as g}from"../chunks/D3tbFfIa.js";import{E as u}from"../chunks/5Rski62M.js";const co='{"title":"Utility functions and classes","local":"utility-functions-and-classes","sections":[{"title":"Constants","local":"constants","sections":[],"depth":2},{"title":"Data Classes","local":"data-classes","sections":[{"title":"Standalone","local":"accelerate.utils.ComputeEnvironment","sections":[],"depth":3},{"title":"Kwargs","local":"accelerate.AutocastKwargs","sections":[],"depth":3}],"depth":2},{"title":"Plugins","local":"accelerate.DeepSpeedPlugin","sections":[],"depth":2},{"title":"Configurations","local":"accelerate.utils.BnbQuantizationConfig","sections":[],"depth":2},{"title":"Environmental Variables","local":"environmental-variables","sections":[],"depth":2},{"title":"Data Manipulation and Operations","local":"accelerate.utils.broadcast","sections":[],"depth":2},{"title":"Environment Checks","local":"accelerate.utils.is_bf16_available","sections":[],"depth":2},{"title":"Environment Manipulation","local":"accelerate.utils.patch_environment","sections":[],"depth":2},{"title":"Memory","local":"accelerate.find_executable_batch_size","sections":[],"depth":2},{"title":"Modeling","local":"accelerate.utils.calculate_maximum_sizes","sections":[],"depth":2},{"title":"Parallel","local":"accelerate.utils.extract_model_from_parallel","sections":[],"depth":2},{"title":"Random","local":"accelerate.utils.set_seed","sections":[],"depth":2},{"title":"PyTorch XLA","local":"accelerate.utils.install_xla","sections":[],"depth":2},{"title":"Loading model weights","local":"accelerate.load_checkpoint_in_model","sections":[],"depth":2},{"title":"Quantization","local":"accelerate.utils.load_and_quantize_model","sections":[],"depth":2}],"depth":1}';var so=_('<meta name="hf:doc:metadata"/>'),h=_("<p>Example:</p> <!>",1),Mt=_(`<p>All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the | |
| meta device (as it would if initialized within the <code>init_empty_weights</code> context manager).</p>`),Pt=_(`<p>Once loaded across devices, you still need to call <a href="/docs/accelerate/pr_4477/en/package_reference/big_modeling#accelerate.dispatch_model">dispatch_model()</a> on your model to make it able to run. To | |
| group the checkpoint loading and dispatch in one single call, use <a href="/docs/accelerate/pr_4477/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>.</p>`),lo=_("<p>Make sure all processes will reach this instruction otherwise one of your processes will hang forever.</p>"),io=_(`<p></p> <!> <!> <p>Below are a variety of utility functions that 🤗 Accelerate provides, broken down by use-case.</p> <!> <p>Constants used throughout 🤗 Accelerate for reference</p> <p>The following are constants used when utilizing <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator.save_state">Accelerator.save_state()</a></p> <p><code>utils.MODEL_NAME</code>: <code>"pytorch_model"</code> <code>utils.OPTIMIZER_NAME</code>: <code>"optimizer"</code> <code>utils.RNG_STATE_NAME</code>: <code>"random_states"</code> <code>utils.SCALER_NAME</code>: <code>"scaler.pt</code> <code>utils.SCHEDULER_NAME</code>: <code>"scheduler</code></p> <p>The following are constants used when utilizing <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator.save_model">Accelerator.save_model()</a></p> <p><code>utils.WEIGHTS_NAME</code>: <code>"pytorch_model.bin"</code> <code>utils.SAFE_WEIGHTS_NAME</code>: <code>"model.safetensors"</code> <code>utils.WEIGHTS_INDEX_NAME</code>: <code>"pytorch_model.bin.index.json"</code> <code>utils.SAFE_WEIGHTS_INDEX_NAME</code>: <code>"model.safetensors.index.json"</code></p> <!> <p>These are basic dataclasses used throughout 🤗 Accelerate and they can be passed in as parameters.</p> <!> <p>These are standalone dataclasses used for checks, such as the type of distributed system being used</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a type of the compute environment.</p> <p>Values:</p> <ul><li><strong>LOCAL_MACHINE</strong> — private/custom cluster hardware.</li> <li><strong>AMAZON_SAGEMAKER</strong> — Amazon SageMaker as compute environment.</li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a type of distributed environment.</p> <p>Values:</p> <ul><li><strong>NO</strong> — Not a distributed environment, just a single process.</li> <li><strong>MULTI_CPU</strong> — Distributed on multiple CPU nodes.</li> <li><strong>MULTI_GPU</strong> — Distributed on multiple GPUs.</li> <li><strong>MULTI_MLU</strong> — Distributed on multiple MLUs.</li> <li><strong>MULTI_SDAA</strong> — Distributed on multiple SDAAs.</li> <li><strong>MULTI_MUSA</strong> — Distributed on multiple MUSAs.</li> <li><strong>MULTI_NPU</strong> — Distributed on multiple NPUs.</li> <li><strong>MULTI_XPU</strong> — Distributed on multiple XPUs.</li> <li><strong>MULTI_HPU</strong> — Distributed on multiple HPUs.</li> <li><strong>MULTI_NEURON</strong> — Distributed on multiple Neuron cores.</li> <li><strong>DEEPSPEED</strong> — Using DeepSpeed.</li> <li><strong>FSDP</strong> — Using Fully Sharded Data Parallelism (FSDP).</li> <li><strong>XLA</strong> — Using TorchXLA.</li> <li><strong>MEGATRON_LM</strong> — Using Megatron-LM.</li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a dynamo backend (see <a href="https://pytorch.org/docs/stable/torch.compiler.html" rel="nofollow">https://pytorch.org/docs/stable/torch.compiler.html</a>).</p> <p>Values:</p> <ul><li><strong>NO</strong> — Do not use torch dynamo.</li> <li><strong>EAGER</strong> — Uses PyTorch to run the extracted GraphModule. This is quite useful in debugging TorchDynamo | |
| issues.</li> <li><strong>AOT_EAGER</strong> — Uses AotAutograd with no compiler, i.e, just using PyTorch eager for the AotAutograd’s | |
| extracted forward and backward graphs. This is useful for debugging, and unlikely to give speedups.</li> <li><strong>INDUCTOR</strong> — Uses TorchInductor backend with AotAutograd and cudagraphs by leveraging codegened Triton | |
| kernels. <a href="https://dev-discuss.pytorch.org/t/torchinductor-a-pytorch-native-compiler-with-define-by-run-ir-and-symbolic-shapes/747" rel="nofollow">Read | |
| more</a></li> <li><strong>AOT_TS_NVFUSER</strong> — nvFuser with AotAutograd/TorchScript. <a href="https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593" rel="nofollow">Read | |
| more</a></li> <li><strong>NVPRIMS_NVFUSER</strong> — nvFuser with PrimTorch. <a href="https://dev-discuss.pytorch.org/t/tracing-with-primitives-update-1-nvfuser-and-its-primitives/593" rel="nofollow">Read | |
| more</a></li> <li><strong>CUDAGRAPHS</strong> — cudagraphs with AotAutograd. <a href="https://github.com/pytorch/torchdynamo/pull/757" rel="nofollow">Read more</a></li> <li><strong>OFI</strong> — Uses Torchscript optimize_for_inference. Inference only. <a href="https://pytorch.org/docs/stable/generated/torch.jit.optimize_for_inference.html" rel="nofollow">Read | |
| more</a></li> <li><strong>FX2TRT</strong> — Uses Nvidia TensorRT for inference optimizations. Inference only. <a href="https://github.com/pytorch/TensorRT/blob/master/docsrc/tutorials/getting_started_with_fx_path.rst" rel="nofollow">Read | |
| more</a></li> <li><strong>ONNXRT</strong> — Uses ONNXRT for inference on CPU/GPU. Inference only. <a href="https://onnxruntime.ai/" rel="nofollow">Read more</a></li> <li><strong>TENSORRT</strong> — Uses ONNXRT to run TensorRT for inference optimizations. <a href="https://github.com/onnx/onnx-tensorrt" rel="nofollow">Read | |
| more</a></li> <li><strong>AOT_TORCHXLA_TRACE_ONCE</strong> — Uses Pytorch/XLA with TorchDynamo optimization, for training. <a href="https://github.com/pytorch/xla/blob/r2.0/docs/dynamo.md" rel="nofollow">Read | |
| more</a></li> <li><strong>TORCHXLA_TRACE_ONCE</strong> — Uses Pytorch/XLA with TorchDynamo optimization, for inference. <a href="https://github.com/pytorch/xla/blob/r2.0/docs/dynamo.md" rel="nofollow">Read | |
| more</a></li> <li><strong>TVM</strong> — Uses Apache TVM for inference optimizations. <a href="https://tvm.apache.org/" rel="nofollow">Read more</a></li> <li><strong>HPU_BACKEND</strong> — Uses HPU backend for inference optimizations.</li> <li><strong>NEURON</strong> — Uses AWS Neuron backend for Trainium/Inferentia.</li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a type of supported experiment tracker</p> <p>Values:</p> <ul><li><strong>ALL</strong> — all available trackers in the environment that are supported</li> <li><strong>TENSORBOARD</strong> — TensorBoard as an experiment tracker</li> <li><strong>WANDB</strong> — wandb as an experiment tracker</li> <li><strong>TRACKIO</strong> — trackio as an experiment tracker</li> <li><strong>COMETML</strong> — comet_ml as an experiment tracker</li> <li><strong>MLFLOW</strong> — mlflow as an experiment tracker</li> <li><strong>CLEARML</strong> — clearml as an experiment tracker</li> <li><strong>DVCLIVE</strong> — dvclive as an experiment tracker</li> <li><strong>SWANLAB</strong> — swanlab as an experiment tracker</li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a type of precision used on floating point values</p> <p>Values:</p> <ul><li><strong>NO</strong> — using full precision (FP32)</li> <li><strong>FP16</strong> — using half precision</li> <li><strong>BF16</strong> — using brain floating point precision</li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Represents a type of distributed environment.</p> <p>Values:</p> <ul><li><strong>NO</strong> — Not a distributed environment, just a single process.</li> <li><strong>DATA_PARALLEL</strong> — using sagemaker distributed data parallelism.</li> <li><strong>MODEL_PARALLEL</strong> — using sagemaker distributed model parallelism.</li></ul></div> <!> <p>These are configurable arguments for specific interactions throughout the PyTorch ecosystem that Accelerate handles under the hood.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Use this object in your <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> to customize how <code>torch.autocast</code> behaves. Please refer to the | |
| documentation of this <a href="https://pytorch.org/docs/stable/amp.html#torch.autocast" rel="nofollow">context manager</a> for more | |
| information on each argument.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Use this object in your <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> to customize how your model is wrapped in a <code>torch.nn.parallel.DistributedDataParallel</code>. Please refer to the documentation of this <a href="https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html" rel="nofollow">wrapper</a> for more | |
| information on each argument.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Use this object in your <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> to customize the behavior of mixed precision, specifically how the <code>torch.amp.GradScaler</code> or <code>torch.cuda.amp.GradScaler</code> used is created. Please refer to the documentation of this <a href="https://pytorch.org/docs/stable/amp.html?highlight=gradscaler" rel="nofollow">scaler</a> for more information on each argument.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Use this object in your <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> to customize the initialization of the distributed processes. Please refer | |
| to the documentation of this <a href="https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group" rel="nofollow">method</a> for more | |
| information on each argument.</p> <p>Note: If <code>timeout</code> is set to <code>None</code>, the default will be based upon how <code>backend</code> is set.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Internal mixin that implements a <code>to_kwargs()</code> method for a dataclass.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Returns a dictionary containing the attributes with values different from the default of this class.</p></div></div> <!> <p>These are plugins that can be passed to the <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> object. While they are defined elsewhere in the documentation, | |
| for convenience all of them are available to see here:</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This plugin is used to integrate DeepSpeed.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Process the DeepSpeed config with the values from the kwargs.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Sets the HfDeepSpeedWeakref to use the current deepspeed plugin configuration</p></div></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This plugin is used to enable fully sharded data parallelism.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Given <code>model</code>, creates an <code>auto_wrap_policy</code> based on the passed in policy and if we can use the <code>transformer_cls_to_wrap</code></p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Sets the mixed precision policy for FSDP</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Set the state dict config based on the <code>StateDictType</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Validates the mixed precision policy, abstracted away to not bring in the imports if not needed.</p></div></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A plugin to configure gradient accumulation behavior. You can only pass one of <code>gradient_accumulation_plugin</code> or <code>gradient_accumulation_steps</code> to <a href="/docs/accelerate/pr_4477/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>. Passing both raises an error.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Plugin for Megatron-LM to enable tensor, pipeline, sequence and data parallelism. Also to enable selective | |
| activation recomputation and optimized fused kernels.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This plugin is used to compile a model with PyTorch 2.0</p></div> <!> <p>These are classes which can be configured and passed through to the appropriate integration</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A plugin to enable BitsAndBytes 4bit and 8bit quantization</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for dataloader-related items when calling <code>accelerator.prepare</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Configuration for the Accelerator object based on inner-project needs.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Sets <code>self.project_dir</code> and <code>self.logging_dir</code> to the appropriate values.</p></div></div> <!> <p>These are environmental variables that can be enabled for different use cases</p> <ul><li><code>ACCELERATE_DEBUG_MODE</code> (<code>str</code>): Whether to run accelerate in debug mode. More info available <a href="../basic_tutorials/troubleshooting#hanging-code-and-timeout-errors">here</a>.</li></ul> <!> <p>These include data operations that mimic the same <code>torch</code> ops but can be used on distributed processes.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively broadcast tensor in a nested list/tuple/dictionary of tensors to all devices.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Broadcast a list of picklable objects from one process to the others.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively concatenate the tensors in a nested list/tuple/dictionary of lists of tensors with the same shape. | |
| If there is only a single batch of data, it is returned as-is.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively converts the elements nested list/tuple/dictionary of tensors in FP16/BF16 precision to FP32.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively gather tensor in a nested list/tuple/dictionary of tensors from all devices.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively gather object in a nested list/tuple/dictionary of objects from all devices.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A generic helper which will initialize the correct <code>GradScaler</code> implementation based on the environment and return | |
| it.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Return a context manager for autocasting mixed precision</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively finds tensors in a nested list/tuple/dictionary and converts them to a list of numbers.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so they | |
| can safely be gathered.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively apply a function on a data structure that is a nested list/tuple/dictionary of a given base type.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively reduce the tensors in a nested list/tuple/dictionary of lists of tensors across all processes by the | |
| mean of a given operation.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively takes a slice in a nested list/tuple/dictionary of tensors.</p></div> <!> <p>These functionalities check the state of the current working environment including information about the operating system itself, what it can support, and if particular dependencies are installed.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Checks if bf16 is supported, optionally ignoring the TPU</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Checks if MPS device is available. The minimum version required is 1.12.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Checks if <code>torch_npu</code> is installed and potentially if a NPU is in the environment</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compares the current PyTorch version to a given reference with an operation.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Check if <code>torch_xla</code> is available. To train a native pytorch job in an environment with torch xla installed, set | |
| the USE_TORCH_XLA to false.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Checks if XPU acceleration is available via stock PyTorch (>=2.7) and | |
| potentially if a XPU is in the environment</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A context manager that will add each keyword argument passed to <code>os.environ</code> and remove them when exiting.</p> <p>Will convert the values in <code>kwargs</code> to strings and upper-case all the keys.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A context manager that will temporarily clear environment variables.</p> <p>When this context exits, the previous environment variables will be back.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Creates and saves a basic cluster config to be used on a local machine with potentially multiple GPUs. Will also | |
| set CPU if it is a CPU-only machine.</p></div> <p>When setting up 🤗 Accelerate for the first time, rather than running <code>accelerate config</code> [~utils.write_basic_config] can be used as an alternative for quick configuration.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Assigns the current process to a specific NUMA node. Ideally most efficient when having at least 2 cpus per node.</p> <p>This result is cached between calls. If you want to override it, please use <code>accelerate.utils.environment.override_numa_afifnity</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Overrides whatever NUMA affinity is set for the current process. This is very taxing and requires recalculating the | |
| affinity to set, ideally you should use <code>utils.environment.set_numa_affinity</code> instead.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Decorator to clean up accelerate environment variables set by the decorated class or function.</p> <p>In some circumstances, calling certain classes or functions can result in accelerate env vars being set and not | |
| being cleaned up afterwards. As an example, when calling:</p> <p>TrainingArguments(fp16=True, …)</p> <p>The following env var will be set:</p> <p>ACCELERATE_MIXED_PRECISION=fp16</p> <p>This can affect subsequent code, since the env var takes precedence over TrainingArguments(fp16=False). This is | |
| especially relevant for unit testing, where we want to avoid the individual tests to have side effects on one | |
| another. Decorate the unit test function or whole class with this decorator to ensure that after each test, the env | |
| vars are cleaned up. This works for both unittest.TestCase and normal classes (pytest); it also works when | |
| decorating the parent class.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A basic decorator that will try to execute <code>function</code>. If it fails from exceptions related to out-of-memory or | |
| CUDNN, the batch size is multiplied by 0.9 and passed to <code>function</code></p> <p><code>function</code> must take in a <code>batch_size</code> parameter as its first argument.</p> <!></div> <!> <p>These utilities relate to interacting with PyTorch models</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Computes the total size of the model and its largest layer</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute the size of each submodule of a given model.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Extract a model from its distributed containers.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute a <code>max_memory</code> dictionary for <a href="/docs/accelerate/pr_4477/en/package_reference/big_modeling#accelerate.infer_auto_device_map">infer_auto_device_map()</a> that will balance the use of each available GPU.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Utility function that will scan a list of named modules and return the maximum size used by one full layer. The | |
| definition of a layer being:</p> <ul><li>a module with no direct children (just parameters and buffers)</li> <li>a module whose class name is in the list <code>no_split_module_classes</code></li></ul></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute a device map for a given model giving priority to GPUs, then offload on CPU and finally offload to disk, | |
| such that:</p> <ul><li>we don’t exceed the memory available of any of the GPU.</li> <li>if offload to the CPU is needed, there is always room left on GPU 0 to put back the layer offloaded on CPU that | |
| has the largest size.</li> <li>if offload to the CPU is needed,we don’t exceed the RAM available on the CPU.</li> <li>if offload to the disk is needed, there is always room left on the CPU to put back the layer offloaded on disk | |
| that has the largest size.</li></ul> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are | |
| loaded.</p> <!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Loads the weights from the offload folder into the model.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Load a checkpoint from a given file. If the checkpoint is in the safetensors format and a device map is passed, the | |
| weights can be fast-loaded directly on the GPU.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Offload a state dict in a given folder.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Reties tied parameters in a given model if the link was broken (for instance when adding hooks).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A helper function to set a given tensor (parameter of buffer) of a module on a specific device (note that doing <code>param.to(device)</code> creates a new tensor not linked to the parameter, which is why we need this function).</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Traverse the model in bottom-up order and return the children modules in that order.</p></div> <!> <p>These include general utilities that should be used when working in parallel.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Extract a model from its distributed containers.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Save the data to disk. Use in place of <code>torch.save()</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compatible drop-in replacement of <code>torch.load()</code> which allows for <code>weights_only</code> to be used if <code>torch</code> version is | |
| 2.4.0 or higher. Otherwise will ignore the kwarg.</p> <p>Will also add (and then remove) an exception for numpy arrays</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Introduces a blocking point in the script, making sure all processes have reached this point before continuing.</p> <!></div> <!> <p>These utilities relate to setting and synchronizing of all the random states.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Helper function for reproducible behavior to set the seed in <code>random</code>, <code>numpy</code>, <code>torch</code>.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!></div> <!> <p>These include utilities that are useful while using PyTorch with XLA.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Helper function to install appropriate xla wheels based on the <code>torch</code> version in Google Colaboratory.</p> <!></div> <!> <p>These include utilities that are useful to load checkpoints.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are | |
| loaded.</p> <!></div> <!> <p>These include utilities that are useful to quantize model.</p> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>This function will quantize the input model with the associated config passed in <code>bnb_quantization_config</code>. If the | |
| model is in the meta device, we will load and dispatch the weights according to the <code>device_map</code> passed. If the | |
| model is already loaded, we will quantize the model and put the model on the GPU,</p></div> <!> <p></p>`,1);function go(Dt,Lt){oo(Lt,!1),ro(()=>{new URLSearchParams(window.location.search).get("fw")}),$a();var We=io();Ka("m2ngq6",c=>{var l=so();ao(l,"content",co),d(c,l)});var Ee=e(m(We),2);eo(Ee,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var Ie=e(Ee,2);s(Ie,{title:"Utility functions and classes",local:"utility-functions-and-classes",headingTag:"h1"});var Ge=e(Ie,4);s(Ge,{title:"Constants",local:"constants",headingTag:"h2"});var Je=e(Ge,12);s(Je,{title:"Data Classes",local:"data-classes",headingTag:"h2"});var qe=e(Je,4);s(qe,{title:"Standalone",local:"accelerate.utils.ComputeEnvironment",headingTag:"h3"});var f=e(qe,4),Ft=a(f);t(Ft,{name:"class accelerate.utils.ComputeEnvironment",anchor:"accelerate.utils.ComputeEnvironment",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L600",parameters:[{name:"*values",val:""}]}),r(6),o(f);var b=e(f,2),Ut=a(b);t(Ut,{name:"class accelerate.DistributedType",anchor:"accelerate.DistributedType",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L529",parameters:[{name:"*values",val:""}]}),r(6),o(b);var v=e(b,2),zt=a(v);t(zt,{name:"class accelerate.utils.DynamoBackend",anchor:"accelerate.utils.DynamoBackend",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L615",parameters:[{name:"*values",val:""}]}),r(6),o(v);var y=e(v,2),jt=a(y);t(jt,{name:"class accelerate.utils.LoggerType",anchor:"accelerate.utils.LoggerType",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L670",parameters:[{name:"*values",val:""}]}),r(6),o(y);var w=e(y,2),Ct=a(w);t(Ct,{name:"class accelerate.utils.PrecisionType",anchor:"accelerate.utils.PrecisionType",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L698",parameters:[{name:"*values",val:""}]}),r(6),o(w);var x=e(w,2),At=a(x);t(At,{name:"class accelerate.utils.RNGType",anchor:"accelerate.utils.RNGType",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L714",parameters:[{name:"*values",val:""}]}),o(x);var k=e(x,2),St=a(k);t(St,{name:"class accelerate.utils.SageMakerDistributedType",anchor:"accelerate.utils.SageMakerDistributedType",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L568",parameters:[{name:"*values",val:""}]}),r(6),o(k);var Be=e(k,2);s(Be,{title:"Kwargs",local:"accelerate.AutocastKwargs",headingTag:"h3"});var T=e(Be,4),Oe=a(T);t(Oe,{name:"class accelerate.AutocastKwargs",anchor:"accelerate.AutocastKwargs",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L109",parameters:[{name:"enabled",val:": bool = True"},{name:"cache_enabled",val:": bool | None = None"}]});var Rt=e(Oe,4);u(Rt,{anchor:"accelerate.AutocastKwargs.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwQXV0b2Nhc3RLd2FyZ3MlMEElMEFrd2FyZ3MlMjAlM0QlMjBBdXRvY2FzdEt3YXJncyhjYWNoZV9lbmFibGVkJTNEVHJ1ZSklMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKGt3YXJnc19oYW5kbGVycyUzRCU1Qmt3YXJncyU1RCk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator | |
| <span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> AutocastKwargs | |
| kwargs = AutocastKwargs(cache_enabled=<span class="hljs-literal">True</span>) | |
| accelerator = Accelerator(kwargs_handlers=[kwargs])`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(T);var N=e(T,2),Xe=a(N);t(Xe,{name:"class accelerate.DistributedDataParallelKwargs",anchor:"accelerate.DistributedDataParallelKwargs",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L151",parameters:[{name:"dim",val:": int = 0"},{name:"broadcast_buffers",val:": bool = True"},{name:"bucket_cap_mb",val:": int = 25"},{name:"find_unused_parameters",val:": bool = False"},{name:"check_reduction",val:": bool = False"},{name:"gradient_as_bucket_view",val:": bool = False"},{name:"static_graph",val:": bool = False"},{name:"comm_hook",val:": DDPCommunicationHookType = <DDPCommunicationHookType.NO: 'no'>"},{name:"comm_wrapper",val:": typing.Literal[<DDPCommunicationHookType.NO: 'no'>, <DDPCommunicationHookType.FP16: 'fp16'>, <DDPCommunicationHookType.BF16: 'bf16'>] = <DDPCommunicationHookType.NO: 'no'>"},{name:"comm_state_option",val:": dict = <factory>"}]});var Wt=e(Xe,4);u(Wt,{anchor:"accelerate.DistributedDataParallelKwargs.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWxLd2FyZ3MlMEElMEFrd2FyZ3MlMjAlM0QlMjBEaXN0cmlidXRlZERhdGFQYXJhbGxlbEt3YXJncyhmaW5kX3VudXNlZF9wYXJhbWV0ZXJzJTNEVHJ1ZSklMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKGt3YXJnc19oYW5kbGVycyUzRCU1Qmt3YXJncyU1RCk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator | |
| <span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> DistributedDataParallelKwargs | |
| kwargs = DistributedDataParallelKwargs(find_unused_parameters=<span class="hljs-literal">True</span>) | |
| accelerator = Accelerator(kwargs_handlers=[kwargs])`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(N);var M=e(N,2),Ve=a(M);t(Ve,{name:"class accelerate.GradScalerKwargs",anchor:"accelerate.GradScalerKwargs",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L229",parameters:[{name:"init_scale",val:": float = 65536.0"},{name:"growth_factor",val:": float = 2.0"},{name:"backoff_factor",val:": float = 0.5"},{name:"growth_interval",val:": int = 2000"},{name:"enabled",val:": bool = True"}]});var Et=e(Ve,4);u(Et,{anchor:"accelerate.GradScalerKwargs.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwR3JhZFNjYWxlckt3YXJncyUwQSUwQWt3YXJncyUyMCUzRCUyMEdyYWRTY2FsZXJLd2FyZ3MoYmFja29mZl9mYWN0b3IlM0QwLjI1KSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3Ioa3dhcmdzX2hhbmRsZXJzJTNEJTVCa3dhcmdzJTVEKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator | |
| <span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> GradScalerKwargs | |
| kwargs = GradScalerKwargs(backoff_factor=<span class="hljs-number">0.25</span>) | |
| accelerator = Accelerator(kwargs_handlers=[kwargs])`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(M);var P=e(M,2),Ze=a(P);t(Ze,{name:"class accelerate.InitProcessGroupKwargs",anchor:"accelerate.InitProcessGroupKwargs",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L254",parameters:[{name:"backend",val:": str | None = 'nccl'"},{name:"init_method",val:": str | None = None"},{name:"timeout",val:": datetime.timedelta | None = None"}]});var It=e(Ze,6);u(It,{anchor:"accelerate.InitProcessGroupKwargs.example",children:(c,l)=>{p(c,{code:"ZnJvbSUyMGRhdGV0aW1lJTIwaW1wb3J0JTIwdGltZWRlbHRhJTBBZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwSW5pdFByb2Nlc3NHcm91cEt3YXJncyUwQSUwQWt3YXJncyUyMCUzRCUyMEluaXRQcm9jZXNzR3JvdXBLd2FyZ3ModGltZW91dCUzRHRpbWVkZWx0YShzZWNvbmRzJTNEODAwKSklMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKGt3YXJnc19oYW5kbGVycyUzRCU1Qmt3YXJncyU1RCk=",highlighted:`<span class="hljs-keyword">from</span> datetime <span class="hljs-keyword">import</span> timedelta | |
| <span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator | |
| <span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> InitProcessGroupKwargs | |
| kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=<span class="hljs-number">800</span>)) | |
| accelerator = Accelerator(kwargs_handlers=[kwargs])`,lang:"python",wrap:!1})},$$slots:{default:!0}}),o(P);var D=e(P,2),Ye=a(D);t(Ye,{name:"class accelerate.utils.KwargsHandler",anchor:"accelerate.utils.KwargsHandler",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L64",parameters:[]});var Qe=e(Ye,4),Gt=a(Qe);t(Gt,{name:"to_kwargs",anchor:"accelerate.utils.KwargsHandler.to_kwargs",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L72",parameters:[]}),r(2),o(Qe),o(D);var He=e(D,2);s(He,{title:"Plugins",local:"accelerate.DeepSpeedPlugin",headingTag:"h2"});var L=e(He,4),$e=a(L);t($e,{name:"class accelerate.DeepSpeedPlugin",anchor:"accelerate.DeepSpeedPlugin",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1047",parameters:[{name:"hf_ds_config",val:": Any = None"},{name:"gradient_accumulation_steps",val:": int = None"},{name:"gradient_clipping",val:": float = None"},{name:"zero_stage",val:": int = None"},{name:"is_train_batch_min",val:": bool = True"},{name:"offload_optimizer_device",val:": str = None"},{name:"offload_param_device",val:": str = None"},{name:"offload_optimizer_nvme_path",val:": str = None"},{name:"offload_param_nvme_path",val:": str = None"},{name:"zero3_init_flag",val:": bool = None"},{name:"zero3_save_16bit_model",val:": bool = None"},{name:"transformer_moe_cls_names",val:": str = None"}],parametersDescription:[{anchor:"accelerate.DeepSpeedPlugin.hf_ds_config",description:`<strong>hf_ds_config</strong> (<code>Any</code>, defaults to <code>None</code>) — | |
| Path to DeepSpeed config file or dict or an object of class <code>accelerate.utils.deepspeed.HfDeepSpeedConfig</code>.`,name:"hf_ds_config"},{anchor:"accelerate.DeepSpeedPlugin.gradient_accumulation_steps",description:`<strong>gradient_accumulation_steps</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of steps to accumulate gradients before updating optimizer states. If not set, will use the value | |
| from the <code>Accelerator</code> directly.`,name:"gradient_accumulation_steps"},{anchor:"accelerate.DeepSpeedPlugin.gradient_clipping",description:`<strong>gradient_clipping</strong> (<code>float</code>, defaults to <code>None</code>) — | |
| Enable gradient clipping with value.`,name:"gradient_clipping"},{anchor:"accelerate.DeepSpeedPlugin.zero_stage",description:`<strong>zero_stage</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Possible options are 0, 1, 2, 3. Default will be taken from environment variable.`,name:"zero_stage"},{anchor:"accelerate.DeepSpeedPlugin.is_train_batch_min",description:`<strong>is_train_batch_min</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| If both train & eval dataloaders are specified, this will decide the <code>train_batch_size</code>.`,name:"is_train_batch_min"},{anchor:"accelerate.DeepSpeedPlugin.offload_optimizer_device",description:`<strong>offload_optimizer_device</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Possible options are none|cpu|nvme. Only applicable with ZeRO Stages 2 and 3.`,name:"offload_optimizer_device"},{anchor:"accelerate.DeepSpeedPlugin.offload_param_device",description:`<strong>offload_param_device</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Possible options are none|cpu|nvme. Only applicable with ZeRO Stage 3.`,name:"offload_param_device"},{anchor:"accelerate.DeepSpeedPlugin.offload_optimizer_nvme_path",description:`<strong>offload_optimizer_nvme_path</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3.`,name:"offload_optimizer_nvme_path"},{anchor:"accelerate.DeepSpeedPlugin.offload_param_nvme_path",description:`<strong>offload_param_nvme_path</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Possible options are /nvme|/local_nvme. Only applicable with ZeRO Stage 3.`,name:"offload_param_nvme_path"},{anchor:"accelerate.DeepSpeedPlugin.zero3_init_flag",description:`<strong>zero3_init_flag</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Flag to indicate whether to save 16-bit model. Only applicable with ZeRO Stage-3.`,name:"zero3_init_flag"},{anchor:"accelerate.DeepSpeedPlugin.zero3_save_16bit_model",description:`<strong>zero3_save_16bit_model</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Flag to indicate whether to save 16-bit model. Only applicable with ZeRO Stage-3.`,name:"zero3_save_16bit_model"},{anchor:"accelerate.DeepSpeedPlugin.transformer_moe_cls_names",description:`<strong>transformer_moe_cls_names</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Comma-separated list of Transformers MoE layer class names (case-sensitive). For example, | |
| <code>MixtralSparseMoeBlock</code>, <code>Qwen2MoeSparseMoeBlock</code>, <code>JetMoEAttention</code>, <code>JetMoEBlock</code>, etc.`,name:"transformer_moe_cls_names"}]});var F=e($e,4),Jt=a(F);t(Jt,{name:"deepspeed_config_process",anchor:"accelerate.DeepSpeedPlugin.deepspeed_config_process",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1283",parameters:[{name:"prefix",val:" = ''"},{name:"mismatches",val:" = None"},{name:"config",val:" = None"},{name:"must_match",val:" = True"},{name:"**kwargs",val:""}]}),r(2),o(F);var Ke=e(F,2),qt=a(Ke);t(qt,{name:"select",anchor:"accelerate.DeepSpeedPlugin.select",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1438",parameters:[{name:"_from_accelerator_state",val:": bool = False"}]}),r(2),o(Ke),o(L);var U=e(L,2),et=a(U);t(et,{name:"class accelerate.FullyShardedDataParallelPlugin",anchor:"accelerate.FullyShardedDataParallelPlugin",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1470",parameters:[{name:"fsdp_version",val:": int = None"},{name:"sharding_strategy",val:": str | ForwardRef('torch.distributed.fsdp.ShardingStrategy') = None"},{name:"reshard_after_forward",val:": str | ForwardRef('torch.distributed.fsdp.ShardingStrategy') | bool = None"},{name:"backward_prefetch",val:": str | ForwardRef('torch.distributed.fsdp.BackwardPrefetch') | None = None"},{name:"mixed_precision_policy",val:": dict | str | ForwardRef('torch.distributed.fsdp.MixedPrecision') | ForwardRef('torch.distributed.fsdp.MixedPrecisionPolicy') | None = None"},{name:"auto_wrap_policy",val:": typing.Callable | typing.Literal['transformer_based_wrap', 'size_based_wrap', 'no_wrap'] | None = None"},{name:"cpu_offload",val:": bool | ForwardRef('torch.distributed.fsdp.CPUOffload') | ForwardRef('torch.distributed.fsdp.CPUOffloadPolicy') = None"},{name:"ignored_modules",val:": collections.abc.Iterable[torch.nn.Module] | str | None = None"},{name:"state_dict_type",val:": str | ForwardRef('torch.distributed.fsdp.StateDictType') = None"},{name:"state_dict_config",val:": ForwardRef('torch.distributed.fsdp.FullStateDictConfig') | ForwardRef('torch.distributed.fsdp.ShardedStateDictConfig') | None = None"},{name:"optim_state_dict_config",val:": ForwardRef('torch.distributed.fsdp.FullOptimStateDictConfig') | ForwardRef('torch.distributed.fsdp.ShardedOptimStateDictConfig') | None = None"},{name:"limit_all_gathers",val:": bool = True"},{name:"use_orig_params",val:": bool | None = None"},{name:"param_init_fn",val:": typing.Callable[[torch.nn.Module], NoneType] | None = None"},{name:"sync_module_states",val:": bool | None = None"},{name:"forward_prefetch",val:": bool = None"},{name:"activation_checkpointing",val:": bool = None"},{name:"activation_checkpointing_offload",val:": bool = None"},{name:"cpu_ram_efficient_loading",val:": bool = None"},{name:"transformer_cls_names_to_wrap",val:": list[str] | None = None"},{name:"min_num_params",val:": int | None = None"}],parametersDescription:[{anchor:"accelerate.FullyShardedDataParallelPlugin.fsdp_version",description:`<strong>fsdp_version</strong> (<code>int</code>, defaults to <code>1</code>) — | |
| The version of FSDP to use. Defaults to 1. If set to 2, launcher expects the config to be converted to | |
| FSDP2 format.`,name:"fsdp_version"},{anchor:"accelerate.FullyShardedDataParallelPlugin.sharding_strategy",description:`<strong>sharding_strategy</strong> (<code>Union[str, torch.distributed.fsdp.ShardingStrategy]</code>, defaults to <code>'FULL_SHARD'</code>) — | |
| Sharding strategy to use. Should be either a <code>str</code> or an instance of | |
| <code>torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy</code>. Is deprecated in favor of | |
| <code>reshard_after_forward</code>.`,name:"sharding_strategy"},{anchor:"accelerate.FullyShardedDataParallelPlugin.reshard_after_forward",description:`<strong>reshard_after_forward</strong> (<code>Union[str, torch.distributed.fsdp.ShardingStrategy, bool]</code>, defaults to <code>'FULL_SHARD'</code> for <code>fsdp_version=1</code> and <code>True</code> for <code>fsdp_version=2</code>) — | |
| Sharding strategy to use. Should be a bool if <code>fsdp_version</code> is set to 2 else a <code>str</code> or an instance of | |
| <code>torch.distributed.fsdp.fully_sharded_data_parallel.ShardingStrategy</code>.`,name:"reshard_after_forward"},{anchor:"accelerate.FullyShardedDataParallelPlugin.backward_prefetch",description:`<strong>backward_prefetch</strong> (<code>Union[str, torch.distributed.fsdp.BackwardPrefetch]</code>, defaults to <code>'NO_PREFETCH'</code>) — | |
| Backward prefetch strategy to use. Should be either a <code>str</code> or an instance of | |
| <code>torch.distributed.fsdp.fully_sharded_data_parallel.BackwardPrefetch</code>.`,name:"backward_prefetch"},{anchor:"accelerate.FullyShardedDataParallelPlugin.mixed_precision_policy",description:`<strong>mixed_precision_policy</strong> (<code>Optional[Union[dict, str, torch.distributed.fsdp.MixedPrecision, torch.distributed.fsdp.MixedPrecisionPolicy]]</code>, defaults to <code>None</code>) — | |
| A config to enable mixed precision training with FullyShardedDataParallel. If passing in a <code>dict</code>, it | |
| should have the following keys: <code>param_dtype</code>, <code>reduce_dtype</code>, and <code>buffer_dtype</code>, can be an instance of | |
| <code>torch.distributed.fsdp.MixedPrecisionPolicy</code> if <code>fsdp_version</code> is set to 2. If passing in a <code>str</code>, it | |
| should be one of the following values: fp8, fp16, bf16, fp32, and used to set <code>param_dtype</code>, | |
| <code>reduce_dtype</code>, and <code>buffer_dtype</code>.`,name:"mixed_precision_policy"},{anchor:"accelerate.FullyShardedDataParallelPlugin.auto_wrap_policy",description:"<strong>auto_wrap_policy</strong> (<code>Optional(Union[Callable, Literal["transformer_based_wrap", "size_based_wrap", "no_wrap"]]), defaults to </code>NO_WRAP<code>) -- A callable or string specifying a policy to recursively wrap layers with FSDP. If a string, it must be one of </code>transformer_based_wrap<code>, </code>size_based_wrap<code>, or </code>no_wrap<code>. See </code>torch.distributed.fsdp.wrap.size_based_wrap_policy` for a direction on what it should look like.",name:"auto_wrap_policy"},{anchor:"accelerate.FullyShardedDataParallelPlugin.cpu_offload",description:`<strong>cpu_offload</strong> (<code>Union[bool, torch.distributed.fsdp.CPUOffload, torch.distributed.fsdp.CPUOffloadPolicy]</code>, defaults to <code>False</code>) — | |
| Whether to offload parameters to CPU. Should be either a <code>bool</code> or an instance of | |
| <code>torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffload</code> or | |
| <code>torch.distributed.fsdp.fully_sharded_data_parallel.CPUOffloadPolicy</code> if <code>fsdp_version</code> is set to 2.`,name:"cpu_offload"},{anchor:"accelerate.FullyShardedDataParallelPlugin.ignored_modules",description:`<strong>ignored_modules</strong> (<code>Optional[Union[Iterable[torch.nn.Module], str]]</code>, defaults to <code>None</code>) — | |
| A list of modules to ignore when wrapping with FSDP. When passing a string, will match the modules by name | |
| using regex fullmatch. If <code>fsdp_version</code> is set to 2, the modules are converted to parameters and used.`,name:"ignored_modules"},{anchor:"accelerate.FullyShardedDataParallelPlugin.state_dict_type",description:`<strong>state_dict_type</strong> (<code>Union[str, torch.distributed.fsdp.StateDictType]</code>, defaults to <code>'FULL_STATE_DICT'</code>) — | |
| State dict type to use. If a string, it must be one of <code>full_state_dict</code>, <code>local_state_dict</code>, or | |
| <code>sharded_state_dict</code>.`,name:"state_dict_type"},{anchor:"accelerate.FullyShardedDataParallelPlugin.state_dict_config",description:`<strong>state_dict_config</strong> (<code>Optional[Union[torch.distributed.fsdp.FullStateDictConfig, torch.distributed.fsdp.ShardedStateDictConfig]</code>, defaults to <code>None</code>) — | |
| State dict config to use. Is determined based on the <code>state_dict_type</code> if not passed in.`,name:"state_dict_config"},{anchor:"accelerate.FullyShardedDataParallelPlugin.optim_state_dict_config",description:`<strong>optim_state_dict_config</strong> (<code>Optional[Union[torch.distributed.fsdp.FullOptimStateDictConfig, torch.distributed.fsdp.ShardedOptimStateDictConfig]</code>, defaults to <code>None</code>) — | |
| Optim state dict config to use. Is determined based on the <code>state_dict_type</code> if not passed in.`,name:"optim_state_dict_config"},{anchor:"accelerate.FullyShardedDataParallelPlugin.limit_all_gathers",description:`<strong>limit_all_gathers</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| Whether to have FSDP explicitly synchronizes the CPU thread to prevent too many in-flight all-gathers. This | |
| bool only affects the sharded strategies that schedule all-gathers. Enabling this can help lower the number | |
| of CUDA malloc retries.`,name:"limit_all_gathers"},{anchor:"accelerate.FullyShardedDataParallelPlugin.use_orig_params",description:`<strong>use_orig_params</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to use the original parameters for the optimizer.`,name:"use_orig_params"},{anchor:"accelerate.FullyShardedDataParallelPlugin.param_init_fn",description:`<strong>param_init_fn</strong> (<code>Optional[Callable[[torch.nn.Module], None]</code>, defaults to <code>None</code>) — | |
| A <code>Callable[torch.nn.Module] -> None</code> that specifies how modules that are currently on the meta device | |
| should be initialized onto an actual device. Only applicable when <code>sync_module_states</code> is <code>True</code>. By | |
| default is a <code>lambda</code> which calls <code>to_empty</code> on the module.`,name:"param_init_fn"},{anchor:"accelerate.FullyShardedDataParallelPlugin.sync_module_states",description:`<strong>sync_module_states</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether each individually wrapped FSDP unit should broadcast module parameters from rank 0 to ensure they | |
| are the same across all ranks after initialization. Defaults to <code>False</code> unless <code>cpu_ram_efficient_loading</code> | |
| is <code>True</code>, then will be forcibly enabled.`,name:"sync_module_states"},{anchor:"accelerate.FullyShardedDataParallelPlugin.forward_prefetch",description:`<strong>forward_prefetch</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to have FSDP explicitly prefetches the next upcoming all-gather while executing in the forward | |
| pass. only use with Static graphs.`,name:"forward_prefetch"},{anchor:"accelerate.FullyShardedDataParallelPlugin.activation_checkpointing",description:`<strong>activation_checkpointing</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| A technique to reduce memory usage by clearing activations of certain layers and recomputing them during a | |
| backward pass. Effectively, this trades extra computation time for reduced memory usage.`,name:"activation_checkpointing"},{anchor:"accelerate.FullyShardedDataParallelPlugin.cpu_ram_efficient_loading",description:`<strong>cpu_ram_efficient_loading</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| If True, only the first process loads the pretrained model checkpoint while all other processes have empty | |
| weights. Only applicable for Transformers. When using this, <code>sync_module_states</code> needs to be <code>True</code>.`,name:"cpu_ram_efficient_loading"},{anchor:"accelerate.FullyShardedDataParallelPlugin.transformer_cls_names_to_wrap",description:`<strong>transformer_cls_names_to_wrap</strong> (<code>Optional[List[str]]</code>, defaults to <code>None</code>) — | |
| A list of transformer layer class names to wrap. Only applicable when <code>auto_wrap_policy</code> is | |
| <code>transformer_based_wrap</code>.`,name:"transformer_cls_names_to_wrap"},{anchor:"accelerate.FullyShardedDataParallelPlugin.min_num_params",description:`<strong>min_num_params</strong> (<code>Optional[int]</code>, defaults to <code>None</code>) — | |
| The minimum number of parameters a module must have to be wrapped. Only applicable when <code>auto_wrap_policy</code> | |
| is <code>size_based_wrap</code>.`,name:"min_num_params"}]});var z=e(et,4),Bt=a(z);t(Bt,{name:"set_auto_wrap_policy",anchor:"accelerate.FullyShardedDataParallelPlugin.set_auto_wrap_policy",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1955",parameters:[{name:"model",val:""}]}),r(2),o(z);var j=e(z,2),Ot=a(j);t(Ot,{name:"set_mixed_precision",anchor:"accelerate.FullyShardedDataParallelPlugin.set_mixed_precision",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1995",parameters:[{name:"mixed_precision",val:""},{name:"buffer_autocast",val:" = False"},{name:"override",val:" = False"}]}),r(2),o(j);var C=e(j,2),Xt=a(C);t(Xt,{name:"set_state_dict_type",anchor:"accelerate.FullyShardedDataParallelPlugin.set_state_dict_type",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L1910",parameters:[{name:"state_dict_type",val:" = None"}]}),r(2),o(C);var tt=e(C,2),Vt=a(tt);t(Vt,{name:"validate_mixed_precision_policy",anchor:"accelerate.FullyShardedDataParallelPlugin.validate_mixed_precision_policy",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L2043",parameters:[]}),r(2),o(tt),o(U);var A=e(U,2),at=a(A);t(at,{name:"class accelerate.utils.GradientAccumulationPlugin",anchor:"accelerate.utils.GradientAccumulationPlugin",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L906",parameters:[{name:"num_steps",val:": int = None"},{name:"adjust_scheduler",val:": bool = True"},{name:"sync_with_dataloader",val:": bool = True"},{name:"sync_each_batch",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.GradientAccumulationPlugin.num_steps",description:`<strong>num_steps</strong> (<code>int</code>) — | |
| The number of steps to accumulate gradients for.`,name:"num_steps"},{anchor:"accelerate.utils.GradientAccumulationPlugin.adjust_scheduler",description:`<strong>adjust_scheduler</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether to adjust the scheduler steps to account for the number of steps being accumulated. Should be | |
| <code>True</code> if the used scheduler was not adjusted for gradient accumulation.`,name:"adjust_scheduler"},{anchor:"accelerate.utils.GradientAccumulationPlugin.sync_with_dataloader",description:`<strong>sync_with_dataloader</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether to synchronize setting the gradients when at the end of the dataloader.`,name:"sync_with_dataloader"},{anchor:"accelerate.utils.GradientAccumulationPlugin.sync_each_batch",description:`<strong>sync_each_batch</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to synchronize setting the gradients at each data batch. Setting to <code>True</code> may reduce memory | |
| requirements when using gradient accumulation with distributed training, at expense of speed.`,name:"sync_each_batch"}]});var Zt=e(at,4);u(Zt,{anchor:"accelerate.utils.GradientAccumulationPlugin.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBHcmFkaWVudEFjY3VtdWxhdGlvblBsdWdpbiUwQSUwQWdyYWRpZW50X2FjY3VtdWxhdGlvbl9wbHVnaW4lMjAlM0QlMjBHcmFkaWVudEFjY3VtdWxhdGlvblBsdWdpbihudW1fc3RlcHMlM0QyKSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZ3JhZGllbnRfYWNjdW11bGF0aW9uX3BsdWdpbiUzRGdyYWRpZW50X2FjY3VtdWxhdGlvbl9wbHVnaW4p",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> GradientAccumulationPlugin | |
| gradient_accumulation_plugin = GradientAccumulationPlugin(num_steps=<span class="hljs-number">2</span>) | |
| accelerator = Accelerator(gradient_accumulation_plugin=gradient_accumulation_plugin)`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(A);var S=e(A,2),Yt=a(S);t(Yt,{name:"class accelerate.utils.MegatronLMPlugin",anchor:"accelerate.utils.MegatronLMPlugin",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L2205",parameters:[{name:"tp_degree",val:": int = None"},{name:"pp_degree",val:": int = None"},{name:"use_custom_fsdp",val:": bool = None"},{name:"overlap_cpu_optimizer_d2h_h2d",val:": bool = None"},{name:"no_load_optim",val:": bool = None"},{name:"eod_mask_loss",val:": bool = None"},{name:"no_save_optim",val:": bool = None"},{name:"optimizer_cpu_offload",val:": bool = None"},{name:"use_precision_aware_optimizer",val:": bool = None"},{name:"decoder_last_pipeline_num_layers",val:": int = None"},{name:"recompute_granularity",val:": str = None"},{name:"recompute_method",val:": str = None"},{name:"recompute_num_layers",val:": int = None"},{name:"attention_backend",val:": bool = None"},{name:"expert_model_parallel_size",val:": int = None"},{name:"context_parallel_size",val:": int = None"},{name:"attention_dropout",val:": float = None"},{name:"hidden_dropout",val:": float = None"},{name:"attention_softmax_in_fp32",val:": bool = None"},{name:"expert_tensor_parallel_size",val:": int = None"},{name:"calculate_per_token_loss",val:": bool = None"},{name:"use_rotary_position_embeddings",val:": bool = None"},{name:"num_micro_batches",val:": int = None"},{name:"gradient_clipping",val:": float = None"},{name:"sequence_parallelism",val:": bool = None"},{name:"recompute_activations",val:": bool = None"},{name:"use_distributed_optimizer",val:": bool = None"},{name:"pipeline_model_parallel_split_rank",val:": int = None"},{name:"num_layers_per_virtual_pipeline_stage",val:": int = None"},{name:"is_train_batch_min",val:": str = True"},{name:"train_iters",val:": int = None"},{name:"train_samples",val:": int = None"},{name:"weight_decay_incr_style",val:": str = 'constant'"},{name:"start_weight_decay",val:": float = None"},{name:"end_weight_decay",val:": float = None"},{name:"lr_decay_style",val:": str = 'linear'"},{name:"lr_decay_iters",val:": int = None"},{name:"lr_decay_samples",val:": int = None"},{name:"lr_warmup_iters",val:": int = None"},{name:"lr_warmup_samples",val:": int = None"},{name:"lr_warmup_fraction",val:": float = None"},{name:"min_lr",val:": float = 0"},{name:"consumed_samples",val:": list = None"},{name:"no_wd_decay_cond",val:": typing.Callable | None = None"},{name:"scale_lr_cond",val:": typing.Callable | None = None"},{name:"lr_mult",val:": float = 1.0"},{name:"megatron_dataset_flag",val:": bool = False"},{name:"seq_length",val:": int = None"},{name:"encoder_seq_length",val:": int = None"},{name:"decoder_seq_length",val:": int = None"},{name:"tensorboard_dir",val:": str = None"},{name:"set_all_logging_options",val:": bool = False"},{name:"eval_iters",val:": int = 100"},{name:"eval_interval",val:": int = 1000"},{name:"return_logits",val:": bool = False"},{name:"custom_train_step_class",val:": typing.Any | None = None"},{name:"custom_train_step_kwargs",val:": dict[str, typing.Any] | None = None"},{name:"custom_model_provider_function",val:": typing.Callable | None = None"},{name:"custom_prepare_model_function",val:": typing.Callable | None = None"},{name:"custom_megatron_datasets_provider_function",val:": typing.Callable | None = None"},{name:"custom_get_batch_function",val:": typing.Callable | None = None"},{name:"custom_loss_function",val:": typing.Callable | None = None"},{name:"other_megatron_args",val:": dict[str, typing.Any] | None = None"}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMPlugin.tp_degree",description:`<strong>tp_degree</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Tensor parallelism degree.`,name:"tp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.pp_degree",description:`<strong>pp_degree</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Pipeline parallelism degree.`,name:"pp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.num_micro_batches",description:`<strong>num_micro_batches</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of micro-batches.`,name:"num_micro_batches"},{anchor:"accelerate.utils.MegatronLMPlugin.gradient_clipping",description:`<strong>gradient_clipping</strong> (<code>float</code>, defaults to <code>None</code>) — | |
| Gradient clipping value based on global L2 Norm (0 to disable).`,name:"gradient_clipping"},{anchor:"accelerate.utils.MegatronLMPlugin.sequence_parallelism",description:`<strong>sequence_parallelism</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Enable sequence parallelism.`,name:"sequence_parallelism"},{anchor:"accelerate.utils.MegatronLMPlugin.recompute_activations",description:`<strong>recompute_activations</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Enable selective activation recomputation.`,name:"recompute_activations"},{anchor:"accelerate.utils.MegatronLMPlugin.use_distributed_optimizer",description:`<strong>use_distributed_optimizer</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Enable distributed optimizer.`,name:"use_distributed_optimizer"},{anchor:"accelerate.utils.MegatronLMPlugin.pipeline_model_parallel_split_rank",description:`<strong>pipeline_model_parallel_split_rank</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Rank where encoder and decoder should be split.`,name:"pipeline_model_parallel_split_rank"},{anchor:"accelerate.utils.MegatronLMPlugin.num_layers_per_virtual_pipeline_stage",description:`<strong>num_layers_per_virtual_pipeline_stage</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of layers per virtual pipeline stage.`,name:"num_layers_per_virtual_pipeline_stage"},{anchor:"accelerate.utils.MegatronLMPlugin.is_train_batch_min",description:`<strong>is_train_batch_min</strong> (<code>str</code>, defaults to <code>True</code>) — | |
| If both tran & eval dataloaders are specified, this will decide the <code>micro_batch_size</code>.`,name:"is_train_batch_min"},{anchor:"accelerate.utils.MegatronLMPlugin.train_iters",description:`<strong>train_iters</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Total number of samples to train over all training runs. Note that either train-iters or train-samples | |
| should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.train_samples",description:`<strong>train_samples</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Total number of samples to train over all training runs. Note that either train-iters or train-samples | |
| should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.weight_decay_incr_style",description:`<strong>weight_decay_incr_style</strong> (<code>str</code>, defaults to <code>'constant'</code>) — | |
| Weight decay increment function. choices=[“constant”, “linear”, “cosine”].`,name:"weight_decay_incr_style"},{anchor:"accelerate.utils.MegatronLMPlugin.start_weight_decay",description:`<strong>start_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) — | |
| Initial weight decay coefficient for L2 regularization.`,name:"start_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.end_weight_decay",description:`<strong>end_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) — | |
| End of run weight decay coefficient for L2 regularization.`,name:"end_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_style",description:`<strong>lr_decay_style</strong> (<code>str</code>, defaults to <code>'linear'</code>) — | |
| Learning rate decay function. choices=[‘constant’, ‘linear’, ‘cosine’].`,name:"lr_decay_style"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_iters",description:`<strong>lr_decay_iters</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of iterations for learning rate decay. If None defaults to <code>train_iters</code>.`,name:"lr_decay_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_samples",description:`<strong>lr_decay_samples</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of samples for learning rate decay. If None defaults to <code>train_samples</code>.`,name:"lr_decay_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_iters",description:`<strong>lr_warmup_iters</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of iterations to linearly warmup learning rate over.`,name:"lr_warmup_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_samples",description:`<strong>lr_warmup_samples</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Number of samples to linearly warmup learning rate over.`,name:"lr_warmup_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_fraction",description:`<strong>lr_warmup_fraction</strong> (<code>float</code>, defaults to <code>None</code>) — | |
| Fraction of lr-warmup-(iters/samples) to linearly warmup learning rate over.`,name:"lr_warmup_fraction"},{anchor:"accelerate.utils.MegatronLMPlugin.min_lr",description:`<strong>min_lr</strong> (<code>float</code>, defaults to <code>0</code>) — | |
| Minimum value for learning rate. The scheduler clip values below this threshold.`,name:"min_lr"},{anchor:"accelerate.utils.MegatronLMPlugin.consumed_samples",description:`<strong>consumed_samples</strong> (<code>List</code>, defaults to <code>None</code>) — | |
| Number of samples consumed in the same order as the dataloaders to <code>accelerator.prepare</code> call.`,name:"consumed_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.no_wd_decay_cond",description:`<strong>no_wd_decay_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Condition to disable weight decay.`,name:"no_wd_decay_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.scale_lr_cond",description:`<strong>scale_lr_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Condition to scale learning rate.`,name:"scale_lr_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_mult",description:`<strong>lr_mult</strong> (<code>float</code>, defaults to <code>1.0</code>) — | |
| Learning rate multiplier.`,name:"lr_mult"},{anchor:"accelerate.utils.MegatronLMPlugin.megatron_dataset_flag",description:`<strong>megatron_dataset_flag</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether the format of dataset follows Megatron-LM Indexed/Cached/MemoryMapped format.`,name:"megatron_dataset_flag"},{anchor:"accelerate.utils.MegatronLMPlugin.seq_length",description:`<strong>seq_length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum sequence length to process.`,name:"seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.encoder_seq_length",description:`<strong>encoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum sequence length to process for the encoder.`,name:"encoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.decoder_seq_length",description:`<strong>decoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| Maximum sequence length to process for the decoder.`,name:"decoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.tensorboard_dir",description:`<strong>tensorboard_dir</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Path to save tensorboard logs.`,name:"tensorboard_dir"},{anchor:"accelerate.utils.MegatronLMPlugin.set_all_logging_options",description:`<strong>set_all_logging_options</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to set all logging options.`,name:"set_all_logging_options"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_iters",description:`<strong>eval_iters</strong> (<code>int</code>, defaults to <code>100</code>) — | |
| Number of iterations to run for evaluation validation/test for.`,name:"eval_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_interval",description:`<strong>eval_interval</strong> (<code>int</code>, defaults to <code>1000</code>) — | |
| Interval between running evaluation on validation set.`,name:"eval_interval"},{anchor:"accelerate.utils.MegatronLMPlugin.return_logits",description:`<strong>return_logits</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether to return logits from the model.`,name:"return_logits"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_class",description:`<strong>custom_train_step_class</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom train step class.`,name:"custom_train_step_class"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_kwargs",description:`<strong>custom_train_step_kwargs</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom train step kwargs.`,name:"custom_train_step_kwargs"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_model_provider_function",description:`<strong>custom_model_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom model provider function.`,name:"custom_model_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_prepare_model_function",description:`<strong>custom_prepare_model_function</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom prepare model function.`,name:"custom_prepare_model_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_megatron_datasets_provider_function",description:`<strong>custom_megatron_datasets_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom megatron train_valid_test datasets provider function.`,name:"custom_megatron_datasets_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_get_batch_function",description:`<strong>custom_get_batch_function</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom get batch function.`,name:"custom_get_batch_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_loss_function",description:`<strong>custom_loss_function</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Custom loss function.`,name:"custom_loss_function"},{anchor:"accelerate.utils.MegatronLMPlugin.other_megatron_args",description:`<strong>other_megatron_args</strong> (<code>Optional</code>, defaults to <code>None</code>) — | |
| Other Megatron-LM arguments. Please refer Megatron-LM.`,name:"other_megatron_args"}]}),r(2),o(S);var R=e(S,2),Qt=a(R);t(Qt,{name:"class accelerate.utils.TorchDynamoPlugin",anchor:"accelerate.utils.TorchDynamoPlugin",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L958",parameters:[{name:"backend",val:": DynamoBackend = None"},{name:"mode",val:": str = None"},{name:"fullgraph",val:": bool = None"},{name:"dynamic",val:": bool = None"},{name:"options",val:": Any = None"},{name:"disable",val:": bool = False"},{name:"use_regional_compilation",val:": bool = None"}],parametersDescription:[{anchor:"accelerate.utils.TorchDynamoPlugin.backend",description:`<strong>backend</strong> (<code>DynamoBackend</code>, defaults to <code>None</code>) — | |
| A valid Dynamo backend. See <a href="https://pytorch.org/docs/stable/torch.compiler.html" rel="nofollow">https://pytorch.org/docs/stable/torch.compiler.html</a> for more details.`,name:"backend"},{anchor:"accelerate.utils.TorchDynamoPlugin.mode",description:`<strong>mode</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| Possible options are ‘default’, ‘reduce-overhead’ or ‘max-autotune’.`,name:"mode"},{anchor:"accelerate.utils.TorchDynamoPlugin.fullgraph",description:`<strong>fullgraph</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Whether it is ok to break model into several subgraphs.`,name:"fullgraph"},{anchor:"accelerate.utils.TorchDynamoPlugin.dynamic",description:`<strong>dynamic</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Whether to use dynamic shape for tracing.`,name:"dynamic"},{anchor:"accelerate.utils.TorchDynamoPlugin.options",description:`<strong>options</strong> (<code>Any</code>, defaults to <code>None</code>) — | |
| A dictionary of options to pass to the backend.`,name:"options"},{anchor:"accelerate.utils.TorchDynamoPlugin.disable",description:`<strong>disable</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Turn torch.compile() into a no-op for testing`,name:"disable"},{anchor:"accelerate.utils.TorchDynamoPlugin.use_regional_compilation",description:`<strong>use_regional_compilation</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| Use it to reduce the cold start compilation time of torch.compile() by targeting repeated blocks of the | |
| same class and compiling them sequentially to hit the compiler’s cache. For example, in <code>GPT2LMHeadModel</code>, | |
| the repeated block/class is <code>GPT2Block</code>, and can be accessed as <code>model.transformer.h[0]</code>. The rest of the | |
| model (e.g model.lm_head) is compiled separately.`,name:"use_regional_compilation"}]}),r(2),o(R);var ot=e(R,2);s(ot,{title:"Configurations",local:"accelerate.utils.BnbQuantizationConfig",headingTag:"h2"});var W=e(ot,4),Ht=a(W);t(Ht,{name:"class accelerate.utils.BnbQuantizationConfig",anchor:"accelerate.utils.BnbQuantizationConfig",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L2944",parameters:[{name:"load_in_8bit",val:": bool = False"},{name:"llm_int8_threshold",val:": float = 6.0"},{name:"load_in_4bit",val:": bool = False"},{name:"bnb_4bit_quant_type",val:": str = 'fp4'"},{name:"bnb_4bit_use_double_quant",val:": bool = False"},{name:"bnb_4bit_compute_dtype",val:": str = 'fp16'"},{name:"torch_dtype",val:": dtype = None"},{name:"skip_modules",val:": list = None"},{name:"keep_in_fp32_modules",val:": list = None"}],parametersDescription:[{anchor:"accelerate.utils.BnbQuantizationConfig.load_in_8bit",description:`<strong>load_in_8bit</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Enable 8bit quantization.`,name:"load_in_8bit"},{anchor:"accelerate.utils.BnbQuantizationConfig.llm_int8_threshold",description:`<strong>llm_int8_threshold</strong> (<code>float</code>, defaults to <code>6.0</code>) — | |
| Value of the outliner threshold. Only relevant when <code>load_in_8bit=True</code>.`,name:"llm_int8_threshold"},{anchor:"accelerate.utils.BnbQuantizationConfig.load_in_4bit",description:`<strong>load_in_4bit</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Enable 4bit quantization.`,name:"load_in_4bit"},{anchor:"accelerate.utils.BnbQuantizationConfig.bnb_4bit_quant_type",description:`<strong>bnb_4bit_quant_type</strong> (<code>str</code>, defaults to <code>fp4</code>) — | |
| Set the quantization data type in the <code>bnb.nn.Linear4Bit</code> layers. Options are {‘fp4’,‘np4’}.`,name:"bnb_4bit_quant_type"},{anchor:"accelerate.utils.BnbQuantizationConfig.bnb_4bit_use_double_quant",description:`<strong>bnb_4bit_use_double_quant</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Enable nested quantization where the quantization constants from the first quantization are quantized | |
| again.`,name:"bnb_4bit_use_double_quant"},{anchor:"accelerate.utils.BnbQuantizationConfig.bnb_4bit_compute_dtype",description:`<strong>bnb_4bit_compute_dtype</strong> (<code>bool</code>, defaults to <code>fp16</code>) — | |
| This sets the computational type which might be different than the input time. For example, inputs might be | |
| fp32, but computation can be set to bf16 for speedups. Options are {‘fp32’,‘fp16’,‘bf16’}.`,name:"bnb_4bit_compute_dtype"},{anchor:"accelerate.utils.BnbQuantizationConfig.torch_dtype",description:`<strong>torch_dtype</strong> (<code>torch.dtype</code>, defaults to <code>None</code>) — | |
| This sets the dtype of the remaining non quantized layers. <code>bitsandbytes</code> library suggests to set the value | |
| to <code>torch.float16</code> for 8 bit model and use the same dtype as the compute dtype for 4 bit model.`,name:"torch_dtype"},{anchor:"accelerate.utils.BnbQuantizationConfig.skip_modules",description:`<strong>skip_modules</strong> (<code>List[str]</code>, defaults to <code>None</code>) — | |
| An explicit list of the modules that we don’t quantize. The dtype of these modules will be <code>torch_dtype</code>.`,name:"skip_modules"},{anchor:"accelerate.utils.BnbQuantizationConfig.keep_in_fp32_modules",description:`<strong>keep_in_fp32_modules</strong> (<code>List</code>, defaults to <code>None</code>) — | |
| An explicit list of the modules that we don’t quantize. We keep them in <code>torch.float32</code>.`,name:"keep_in_fp32_modules"}]}),r(2),o(W);var E=e(W,2),$t=a(E);t($t,{name:"class accelerate.DataLoaderConfiguration",anchor:"accelerate.DataLoaderConfiguration",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L748",parameters:[{name:"split_batches",val:": bool = False"},{name:"dispatch_batches",val:": bool = None"},{name:"even_batches",val:": bool = True"},{name:"use_seedable_sampler",val:": bool = False"},{name:"data_seed",val:": int = None"},{name:"non_blocking",val:": bool = False"},{name:"use_stateful_dataloader",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.DataLoaderConfiguration.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether or not the accelerator should split the batches yielded by the dataloaders across the devices. If | |
| <code>True</code>, the actual batch size used will be the same on any kind of distributed processes, but it must be a | |
| round multiple of <code>num_processes</code> you are using. If <code>False</code>, actual batch size used will be the one set in | |
| your script multiplied by the number of processes.`,name:"split_batches"},{anchor:"accelerate.DataLoaderConfiguration.dispatch_batches",description:`<strong>dispatch_batches</strong> (<code>bool</code>, defaults to <code>None</code>) — | |
| If set to <code>True</code>, the dataloader prepared by the Accelerator is only iterated through on the main process | |
| and then the batches are split and broadcast to each process. Will default to <code>True</code> for <code>DataLoader</code> whose | |
| underlying dataset is an <code>IterableDataset</code>, <code>False</code> otherwise.`,name:"dispatch_batches"},{anchor:"accelerate.DataLoaderConfiguration.even_batches",description:`<strong>even_batches</strong> (<code>bool</code>, defaults to <code>True</code>) — | |
| If set to <code>True</code>, in cases where the total batch size across all processes does not exactly divide the | |
| dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among | |
| all workers.`,name:"even_batches"},{anchor:"accelerate.DataLoaderConfiguration.use_seedable_sampler",description:`<strong>use_seedable_sampler</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether or not use a fully seedable random sampler (<code>data_loader.SeedableRandomSampler</code>). Ensures | |
| training results are fully reproducible using a different sampling technique. While seed-to-seed results | |
| may differ, on average the differences are negligible when using multiple different seeds to compare. | |
| Should also be ran with <a href="/docs/accelerate/pr_4477/en/package_reference/utilities#accelerate.utils.set_seed">set_seed()</a> for the best results.`,name:"use_seedable_sampler"},{anchor:"accelerate.DataLoaderConfiguration.data_seed",description:`<strong>data_seed</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| The seed to use for the underlying generator when using <code>use_seedable_sampler</code>. If <code>None</code>, the generator | |
| will use the current default seed from torch.`,name:"data_seed"},{anchor:"accelerate.DataLoaderConfiguration.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If set to <code>True</code>, the dataloader prepared by the Accelerator will utilize non-blocking host-to-device | |
| transfers, allowing for better overlap between dataloader communication and computation. Recommended that | |
| the prepared dataloader has <code>pin_memory</code> set to <code>True</code> to work properly.`,name:"non_blocking"},{anchor:"accelerate.DataLoaderConfiguration.use_stateful_dataloader",description:`<strong>use_stateful_dataloader</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If set to <code>True</code>, the dataloader prepared by the Accelerator will be backed by | |
| <a href="https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader" rel="nofollow">torchdata.StatefulDataLoader</a>. | |
| This requires <code>torchdata</code> version 0.8.0 or higher that supports StatefulDataLoader to be installed.`,name:"use_stateful_dataloader"}]}),r(2),o(E);var I=e(E,2),rt=a(I);t(rt,{name:"class accelerate.utils.ProjectConfiguration",anchor:"accelerate.utils.ProjectConfiguration",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L843",parameters:[{name:"project_dir",val:": str = None"},{name:"logging_dir",val:": str = None"},{name:"automatic_checkpoint_naming",val:": bool = False"},{name:"total_limit",val:": int = None"},{name:"iteration",val:": int = 0"},{name:"save_on_each_node",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.ProjectConfiguration.project_dir",description:`<strong>project_dir</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| A path to a directory for storing data.`,name:"project_dir"},{anchor:"accelerate.utils.ProjectConfiguration.logging_dir",description:`<strong>logging_dir</strong> (<code>str</code>, defaults to <code>None</code>) — | |
| A path to a directory for storing logs of locally-compatible loggers. If None, defaults to <code>project_dir</code>.`,name:"logging_dir"},{anchor:"accelerate.utils.ProjectConfiguration.automatic_checkpoint_naming",description:`<strong>automatic_checkpoint_naming</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| Whether saved states should be automatically iteratively named.`,name:"automatic_checkpoint_naming"},{anchor:"accelerate.utils.ProjectConfiguration.total_limit",description:`<strong>total_limit</strong> (<code>int</code>, defaults to <code>None</code>) — | |
| The maximum number of total saved states to keep.`,name:"total_limit"},{anchor:"accelerate.utils.ProjectConfiguration.iteration",description:`<strong>iteration</strong> (<code>int</code>, defaults to <code>0</code>) — | |
| The current save iteration.`,name:"iteration"},{anchor:"accelerate.utils.ProjectConfiguration.save_on_each_node",description:`<strong>save_on_each_node</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on | |
| the main one.`,name:"save_on_each_node"}]});var nt=e(rt,4),Kt=a(nt);t(Kt,{name:"set_directories",anchor:"accelerate.utils.ProjectConfiguration.set_directories",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/dataclasses.py#L895",parameters:[{name:"project_dir",val:": str | None = None"}]}),r(2),o(nt),o(I);var ct=e(I,2);s(ct,{title:"Environmental Variables",local:"environmental-variables",headingTag:"h2"});var st=e(ct,6);s(st,{title:"Data Manipulation and Operations",local:"accelerate.utils.broadcast",headingTag:"h2"});var G=e(st,4),ea=a(G);t(ea,{name:"accelerate.utils.broadcast",anchor:"accelerate.utils.broadcast",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L604",parameters:[{name:"tensor",val:""},{name:"from_process",val:": int = 0"}],parametersDescription:[{anchor:"accelerate.utils.broadcast.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to gather.`,name:"tensor"},{anchor:"accelerate.utils.broadcast.from_process",description:`<strong>from_process</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| The process from which to send the data`,name:"from_process"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>tensor</code> with all tensors broadcasted to the proper device.</p> | |
| `}),r(2),o(G);var J=e(G,2),ta=a(J);t(ta,{name:"accelerate.utils.broadcast_object_list",anchor:"accelerate.utils.broadcast_object_list",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L679",parameters:[{name:"object_list",val:""},{name:"from_process",val:": int = 0"}],parametersDescription:[{anchor:"accelerate.utils.broadcast_object_list.object_list",description:`<strong>object_list</strong> (list of picklable objects) — | |
| The list of objects to broadcast. This list will be modified inplace.`,name:"object_list"},{anchor:"accelerate.utils.broadcast_object_list.from_process",description:`<strong>from_process</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| The process from which to send the data.`,name:"from_process"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same list containing the objects from process 0.</p> | |
| `}),r(2),o(J);var q=e(J,2),aa=a(q);t(aa,{name:"accelerate.utils.concatenate",anchor:"accelerate.utils.concatenate",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L723",parameters:[{name:"data",val:""},{name:"dim",val:" = 0"}],parametersDescription:[{anchor:"accelerate.utils.concatenate.data",description:`<strong>data</strong> (nested list/tuple/dictionary of lists of tensors <code>torch.Tensor</code>) — | |
| The data to concatenate.`,name:"data"},{anchor:"accelerate.utils.concatenate.dim",description:`<strong>dim</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| The dimension on which to concatenate.`,name:"dim"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>data</code> with all the tensors concatenated.</p> | |
| `}),r(2),o(q);var B=e(q,2),oa=a(B);t(oa,{name:"accelerate.utils.convert_outputs_to_fp32",anchor:"accelerate.utils.convert_outputs_to_fp32",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L943",parameters:[{name:"model_forward",val:""}]}),o(B);var O=e(B,2),ra=a(O);t(ra,{name:"accelerate.utils.convert_to_fp32",anchor:"accelerate.utils.convert_to_fp32",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L893",parameters:[{name:"tensor",val:""}],parametersDescription:[{anchor:"accelerate.utils.convert_to_fp32.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to convert from FP16/BF16 to FP32.`,name:"tensor"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>tensor</code> with all tensors that were in FP16/BF16 precision converted to FP32.</p> | |
| `}),r(2),o(O);var X=e(O,2),na=a(X);t(na,{name:"accelerate.utils.gather",anchor:"accelerate.utils.gather",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L428",parameters:[{name:"tensor",val:""}],parametersDescription:[{anchor:"accelerate.utils.gather.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to gather.`,name:"tensor"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>tensor</code> with all tensors sent to the proper device.</p> | |
| `}),r(2),o(X);var V=e(X,2),ca=a(V);t(ca,{name:"accelerate.utils.gather_object",anchor:"accelerate.utils.gather_object",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L509",parameters:[{name:"object",val:": Any"}],parametersDescription:[{anchor:"accelerate.utils.gather_object.object",description:`<strong>object</strong> (nested list/tuple/dictionary of picklable object) — | |
| The data to gather.`,name:"object"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>object</code> with all the objects sent to every device.</p> | |
| `}),r(2),o(V);var Z=e(V,2),sa=a(Z);t(sa,{name:"accelerate.utils.get_grad_scaler",anchor:"accelerate.utils.get_grad_scaler",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L2133",parameters:[{name:"distributed_type",val:": DistributedType = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.get_grad_scaler.distributed_type",description:`<strong>distributed_type</strong> (<code>DistributedType</code>, <em>optional</em>, defaults to None) — | |
| The type of distributed environment.`,name:"distributed_type"},{anchor:"accelerate.utils.get_grad_scaler.kwargs",description:`<strong>kwargs</strong> — | |
| Additional arguments for the utilized <code>GradScaler</code> constructor.`,name:"kwargs"}]}),r(2),o(Z);var Y=e(Z,2),la=a(Y);t(la,{name:"accelerate.utils.get_mixed_precision_context_manager",anchor:"accelerate.utils.get_mixed_precision_context_manager",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L2091",parameters:[{name:"native_amp",val:": bool = False"},{name:"autocast_kwargs",val:": AutocastKwargs = None"}],parametersDescription:[{anchor:"accelerate.utils.get_mixed_precision_context_manager.native_amp",description:`<strong>native_amp</strong> (<code>bool</code>, <em>optional</em>, defaults to False) — | |
| Whether mixed precision is actually enabled.`,name:"native_amp"}]}),r(2),o(Y);var Q=e(Y,2),ia=a(Q);t(ia,{name:"accelerate.utils.listify",anchor:"accelerate.utils.listify",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L284",parameters:[{name:"data",val:""}],parametersDescription:[{anchor:"accelerate.utils.listify.data",description:"<strong>data</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — The data from which to convert to regular numbers.",name:"data"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>data</code> with lists of numbers instead of <code>torch.Tensor</code>.</p> | |
| `}),r(2),o(Q);var H=e(Q,2),da=a(H);t(da,{name:"accelerate.utils.pad_across_processes",anchor:"accelerate.utils.pad_across_processes",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L753",parameters:[{name:"tensor",val:""},{name:"dim",val:" = 0"},{name:"pad_index",val:" = 0"},{name:"pad_first",val:" = False"}],parametersDescription:[{anchor:"accelerate.utils.pad_across_processes.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to gather.`,name:"tensor"},{anchor:"accelerate.utils.pad_across_processes.dim",description:`<strong>dim</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| The dimension on which to pad.`,name:"dim"},{anchor:"accelerate.utils.pad_across_processes.pad_index",description:`<strong>pad_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) — | |
| The value with which to pad.`,name:"pad_index"},{anchor:"accelerate.utils.pad_across_processes.pad_first",description:`<strong>pad_first</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to pad at the beginning or the end.`,name:"pad_first"}]}),r(2),o(H);var $=e(H,2),pa=a($);t(pa,{name:"accelerate.utils.recursively_apply",anchor:"accelerate.utils.recursively_apply",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L85",parameters:[{name:"func",val:""},{name:"data",val:""},{name:"*args",val:""},{name:"test_type",val:" = <function is_torch_tensor at 0x7f368e14aa30>"},{name:"error_on_other_type",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.recursively_apply.func",description:`<strong>func</strong> (<code>callable</code>) — | |
| The function to recursively apply.`,name:"func"},{anchor:"accelerate.utils.recursively_apply.data",description:`<strong>data</strong> (nested list/tuple/dictionary of objects accepted by <code>test_type</code>) — | |
| The data on which to apply <code>func</code>`,name:"data"},{anchor:"accelerate.utils.recursively_apply.*args",description:`<strong>*args</strong> — | |
| Positional arguments that will be passed to <code>func</code> when applied on the unpacked data.`,name:"*args"},{anchor:"accelerate.utils.recursively_apply.test_type",description:`<strong>test_type</strong> (<code>Callable[[Any], bool]</code>, <em>optional</em>, defaults to <code>is_torch_tensor</code>) — | |
| The predicate deciding whether <code>func</code> should be applied to a given leaf object.`,name:"test_type"},{anchor:"accelerate.utils.recursively_apply.error_on_other_type",description:`<strong>error_on_other_type</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to return an error or not if after unpacking <code>data</code>, we get on an object for which <code>test_type</code> | |
| returns <code>False</code>. If <code>False</code>, the function will leave such objects unchanged.`,name:"error_on_other_type"},{anchor:"accelerate.utils.recursively_apply.*kwargs",description:`*<strong>*kwargs</strong> (additional keyword arguments, <em>optional</em>) — | |
| Keyword arguments that will be passed to <code>func</code> when applied on the unpacked data.`,name:"*kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>data</code> with <code>func</code> applied to every object for which <code>test_type</code> returns <code>True</code>.</p> | |
| `}),r(2),o($);var K=e($,2),ma=a(K);t(ma,{name:"accelerate.utils.reduce",anchor:"accelerate.utils.reduce",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L849",parameters:[{name:"tensor",val:""},{name:"reduction",val:" = 'mean'"},{name:"scale",val:" = 1.0"}],parametersDescription:[{anchor:"accelerate.utils.reduce.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to reduce.`,name:"tensor"},{anchor:"accelerate.utils.reduce.reduction",description:`<strong>reduction</strong> (<code>str</code>, <em>optional</em>, defaults to <code>"mean"</code>) — | |
| A reduction method. Can be of “mean”, “sum”, “max”, or “none”`,name:"reduction"},{anchor:"accelerate.utils.reduce.scale",description:`<strong>scale</strong> (<code>float</code>, <em>optional</em>) — | |
| A default scaling value to be applied after the reduce, only valid on XLA.`,name:"scale"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>data</code> with all the tensors reduced.</p> | |
| `}),r(2),o(K);var ee=e(K,2),ua=a(ee);t(ua,{name:"accelerate.utils.send_to_device",anchor:"accelerate.utils.send_to_device",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L136",parameters:[{name:"tensor",val:""},{name:"device",val:""},{name:"non_blocking",val:" = False"},{name:"skip_keys",val:" = None"}],parametersDescription:[{anchor:"accelerate.utils.send_to_device.tensor",description:`<strong>tensor</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to send to a given device.`,name:"tensor"},{anchor:"accelerate.utils.send_to_device.device",description:`<strong>device</strong> (<code>torch.device</code>) — | |
| The device to send the data to.`,name:"device"},{anchor:"accelerate.utils.send_to_device.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, the transfer to the device is performed asynchronously, which can overlap | |
| data movement with computation. Only effective when the device supports it (e.g. CUDA).`,name:"non_blocking"},{anchor:"accelerate.utils.send_to_device.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A key or list of keys in a dictionary <code>tensor</code> whose values should not be sent to | |
| the given <code>device</code>. Entries with these keys are left on their original device.`,name:"skip_keys"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>tensor</code> with all tensors sent to the proper device.</p> | |
| `}),r(2),o(ee);var te=e(ee,2),ha=a(te);t(ha,{name:"accelerate.utils.slice_tensors",anchor:"accelerate.utils.slice_tensors",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/operations.py#L703",parameters:[{name:"data",val:""},{name:"tensor_slice",val:""},{name:"process_index",val:" = None"},{name:"num_processes",val:" = None"}],parametersDescription:[{anchor:"accelerate.utils.slice_tensors.data",description:`<strong>data</strong> (nested list/tuple/dictionary of <code>torch.Tensor</code>) — | |
| The data to slice.`,name:"data"},{anchor:"accelerate.utils.slice_tensors.tensor_slice",description:`<strong>tensor_slice</strong> (<code>slice</code>) — | |
| The slice to take.`,name:"tensor_slice"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same data structure as <code>data</code> with all the tensors slices.</p> | |
| `}),r(2),o(te);var lt=e(te,2);s(lt,{title:"Environment Checks",local:"accelerate.utils.is_bf16_available",headingTag:"h2"});var ae=e(lt,4),_a=a(ae);t(_a,{name:"accelerate.utils.is_bf16_available",anchor:"accelerate.utils.is_bf16_available",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/imports.py#L156",parameters:[{name:"ignore_tpu",val:" = False"}]}),r(2),o(ae);var oe=e(ae,2),ga=a(oe);t(ga,{name:"accelerate.utils.is_mps_available",anchor:"accelerate.utils.is_mps_available",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/imports.py#L326",parameters:[{name:"min_version",val:" = '1.12'"}]}),r(2),o(oe);var re=e(oe,2),fa=a(re);t(fa,{name:"accelerate.utils.is_npu_available",anchor:"accelerate.utils.is_npu_available",parameters:[{name:"check_device",val:" = False"}]}),r(2),o(re);var ne=e(re,2),ba=a(ne);t(ba,{name:"accelerate.utils.is_torch_version",anchor:"accelerate.utils.is_torch_version",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/versions.py#L46",parameters:[{name:"operation",val:": str"},{name:"version",val:": str"}],parametersDescription:[{anchor:"accelerate.utils.is_torch_version.operation",description:`<strong>operation</strong> (<code>str</code>) — | |
| A string representation of an operator, such as <code>">"</code> or <code>"<="</code>`,name:"operation"},{anchor:"accelerate.utils.is_torch_version.version",description:`<strong>version</strong> (<code>str</code>) — | |
| A string version of PyTorch`,name:"version"}]}),r(2),o(ne);var ce=e(ne,2),va=a(ce);t(va,{name:"accelerate.utils.is_torch_xla_available",anchor:"accelerate.utils.is_torch_xla_available",parameters:[{name:"check_is_tpu",val:" = False"},{name:"check_is_gpu",val:" = False"}]}),r(2),o(ce);var se=e(ce,2),ya=a(se);t(ya,{name:"accelerate.utils.is_xpu_available",anchor:"accelerate.utils.is_xpu_available",parameters:[{name:"check_device",val:" = False"}]}),r(2),o(se);var it=e(se,2);s(it,{title:"Environment Manipulation",local:"accelerate.utils.patch_environment",headingTag:"h2"});var le=e(it,2),dt=a(le);t(dt,{name:"accelerate.utils.patch_environment",anchor:"accelerate.utils.patch_environment",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/environment.py#L416",parameters:[{name:"**kwargs",val:""}]});var wa=e(dt,6);u(wa,{anchor:"accelerate.utils.patch_environment.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"aW1wb3J0JTIwb3MlMEFmcm9tJTIwYWNjZWxlcmF0ZS51dGlscyUyMGltcG9ydCUyMHBhdGNoX2Vudmlyb25tZW50JTBBJTBBd2l0aCUyMHBhdGNoX2Vudmlyb25tZW50KEZPTyUzRCUyMmJhciUyMiklM0ElMEElMjAlMjAlMjAlMjBwcmludChvcy5lbnZpcm9uJTVCJTIyRk9PJTIyJTVEKSUyMCUyMCUyMyUyMHByaW50cyUyMCUyMmJhciUyMiUwQXByaW50KG9zLmVudmlyb24lNUIlMjJGT08lMjIlNUQpJTIwJTIwJTIzJTIwcmFpc2VzJTIwS2V5RXJyb3I=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> os | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> patch_environment | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> patch_environment(FOO=<span class="hljs-string">"bar"</span>): | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(os.environ[<span class="hljs-string">"FOO"</span>]) <span class="hljs-comment"># prints "bar"</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(os.environ[<span class="hljs-string">"FOO"</span>]) <span class="hljs-comment"># raises KeyError</span>`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(le);var ie=e(le,2),pt=a(ie);t(pt,{name:"accelerate.utils.clear_environment",anchor:"accelerate.utils.clear_environment",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/environment.py#L381",parameters:[]});var xa=e(pt,6);u(xa,{anchor:"accelerate.utils.clear_environment.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"aW1wb3J0JTIwb3MlMEFmcm9tJTIwYWNjZWxlcmF0ZS51dGlscyUyMGltcG9ydCUyMGNsZWFyX2Vudmlyb25tZW50JTBBJTBBb3MuZW52aXJvbiU1QiUyMkZPTyUyMiU1RCUyMCUzRCUyMCUyMmJhciUyMiUwQXdpdGglMjBjbGVhcl9lbnZpcm9ubWVudCgpJTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQob3MuZW52aXJvbiklMEElMjAlMjAlMjAlMjBvcy5lbnZpcm9uJTVCJTIyRk9PJTIyJTVEJTIwJTNEJTIwJTIybmV3X2JhciUyMiUwQSUyMCUyMCUyMCUyMHByaW50KG9zLmVudmlyb24lNUIlMjJGT08lMjIlNUQpJTBBJTBBcHJpbnQob3MuZW52aXJvbiU1QiUyMkZPTyUyMiU1RCk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> os | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> clear_environment | |
| <span class="hljs-meta">>>> </span>os.environ[<span class="hljs-string">"FOO"</span>] = <span class="hljs-string">"bar"</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> clear_environment(): | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(os.environ) | |
| <span class="hljs-meta">... </span> os.environ[<span class="hljs-string">"FOO"</span>] = <span class="hljs-string">"new_bar"</span> | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(os.environ[<span class="hljs-string">"FOO"</span>]) | |
| {} | |
| new_bar | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(os.environ[<span class="hljs-string">"FOO"</span>]) | |
| bar`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(ie);var de=e(ie,2),ka=a(de);t(ka,{name:"accelerate.commands.config.default.write_basic_config",anchor:"accelerate.commands.config.default.write_basic_config",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/commands/config/default.py#L37",parameters:[{name:"mixed_precision",val:" = 'no'"},{name:"save_location",val:": str = '/home/runner/.cache/huggingface/accelerate/default_config.yaml'"}],parametersDescription:[{anchor:"accelerate.commands.config.default.write_basic_config.mixed_precision",description:`<strong>mixed_precision</strong> (<code>str</code>, <em>optional</em>, defaults to “no”) — | |
| Mixed Precision to use. Should be one of “no”, “fp16”, or “bf16”`,name:"mixed_precision"},{anchor:"accelerate.commands.config.default.write_basic_config.save_location",description:`<strong>save_location</strong> (<code>str</code>, <em>optional</em>, defaults to <code>default_json_config_file</code>) — | |
| Optional custom save location. Should be passed to <code>--config_file</code> when using <code>accelerate launch</code>. Default | |
| location is inside the huggingface cache folder (<code>~/.cache/huggingface</code>) but can be overridden by setting | |
| the <code>HF_HOME</code> environmental variable, followed by <code>accelerate/default_config.yaml</code>.`,name:"save_location"}]}),r(2),o(de);var pe=e(de,4),Ta=a(pe);t(Ta,{name:"accelerate.utils.set_numa_affinity",anchor:"accelerate.utils.set_numa_affinity",parameters:[{name:"local_process_index",val:": int"},{name:"verbose",val:": bool | None = None"}],parametersDescription:[{anchor:"accelerate.utils.set_numa_affinity.local_process_index",description:`<strong>local_process_index</strong> (int) — | |
| The index of the current process on the current server.`,name:"local_process_index"},{anchor:"accelerate.utils.set_numa_affinity.verbose",description:`<strong>verbose</strong> (bool, <em>optional</em>) — | |
| Whether to print the new cpu cores assignment for each process. If <code>ACCELERATE_DEBUG_MODE</code> is enabled, will | |
| default to True.`,name:"verbose"}]}),r(4),o(pe);var me=e(pe,2),Na=a(me);t(Na,{name:"accelerate.utils.environment.override_numa_affinity",anchor:"accelerate.utils.environment.override_numa_affinity",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/environment.py#L298",parameters:[{name:"local_process_index",val:": int"},{name:"verbose",val:": bool | None = None"}],parametersDescription:[{anchor:"accelerate.utils.environment.override_numa_affinity.local_process_index",description:`<strong>local_process_index</strong> (int) — | |
| The index of the current process on the current server.`,name:"local_process_index"},{anchor:"accelerate.utils.environment.override_numa_affinity.verbose",description:`<strong>verbose</strong> (bool, <em>optional</em>) — | |
| Whether to log out the assignment of each CPU. If <code>ACCELERATE_DEBUG_MODE</code> is enabled, will default to True.`,name:"verbose"}]}),r(2),o(me);var ue=e(me,2),Ma=a(ue);t(Ma,{name:"accelerate.utils.purge_accelerate_environment",anchor:"accelerate.utils.purge_accelerate_environment",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/environment.py#L453",parameters:[{name:"func_or_cls",val:""}]}),r(12),o(ue);var mt=e(ue,2);s(mt,{title:"Memory",local:"accelerate.find_executable_batch_size",headingTag:"h2"});var he=e(mt,2),ut=a(he);t(ut,{name:"accelerate.find_executable_batch_size",anchor:"accelerate.find_executable_batch_size",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/memory.py#L120",parameters:[{name:"function",val:": callable | None = None"},{name:"starting_batch_size",val:": int = 128"},{name:"reduce_batch_size_fn",val:": callable | None = None"}],parametersDescription:[{anchor:"accelerate.find_executable_batch_size.function",description:`<strong>function</strong> (<code>callable</code>, <em>optional</em>) — | |
| A function to wrap`,name:"function"},{anchor:"accelerate.find_executable_batch_size.starting_batch_size",description:`<strong>starting_batch_size</strong> (<code>int</code>, <em>optional</em>) — | |
| The batch size to try and fit into memory`,name:"starting_batch_size"},{anchor:"accelerate.find_executable_batch_size.reduce_batch_size_fn",description:`<strong>reduce_batch_size_fn</strong> (<code>callable</code>, <em>optional</em>) — | |
| A function to determine the new batch size after an out-of-memory error. If not | |
| provided, the batch size is multiplied by 0.9 on each failure. The function takes | |
| no arguments and should return the new (reduced) batch size as an <code>int</code>.`,name:"reduce_batch_size_fn"}]});var Pa=e(ut,6);u(Pa,{anchor:"accelerate.find_executable_batch_size.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBmaW5kX2V4ZWN1dGFibGVfYmF0Y2hfc2l6ZSUwQSUwQSUwQSU0MGZpbmRfZXhlY3V0YWJsZV9iYXRjaF9zaXplKHN0YXJ0aW5nX2JhdGNoX3NpemUlM0QxMjgpJTBBZGVmJTIwdHJhaW4oYmF0Y2hfc2l6ZSUyQyUyMG1vZGVsJTJDJTIwb3B0aW1pemVyKSUzQSUwQSUyMCUyMCUyMCUyMC4uLiUwQSUwQSUwQXRyYWluKG1vZGVsJTJDJTIwb3B0aW1pemVyKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> find_executable_batch_size | |
| <span class="hljs-meta">>>> </span>@find_executable_batch_size(starting_batch_size=<span class="hljs-number">128</span>) | |
| <span class="hljs-meta">... </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">train</span>(<span class="hljs-params">batch_size, model, optimizer</span>): | |
| <span class="hljs-meta">... </span> ... | |
| <span class="hljs-meta">>>> </span>train(model, optimizer)`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(he);var ht=e(he,2);s(ht,{title:"Modeling",local:"accelerate.utils.calculate_maximum_sizes",headingTag:"h2"});var _e=e(ht,4),Da=a(_e);t(Da,{name:"accelerate.utils.calculate_maximum_sizes",anchor:"accelerate.utils.calculate_maximum_sizes",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L1089",parameters:[{name:"model",val:": Module"}]}),r(2),o(_e);var ge=e(_e,2),La=a(ge);t(La,{name:"accelerate.utils.compute_module_sizes",anchor:"accelerate.utils.compute_module_sizes",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L651",parameters:[{name:"model",val:": Module"},{name:"dtype",val:": str | torch.device | None = None"},{name:"special_dtypes",val:": dict[str, str | torch.device] | None = None"},{name:"buffers_only",val:": bool = False"}]}),r(2),o(ge);var fe=e(ge,2),Fa=a(fe);t(Fa,{name:"accelerate.utils.extract_model_from_parallel",anchor:"accelerate.utils.extract_model_from_parallel",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L264",parameters:[{name:"model",val:""},{name:"keep_fp32_wrapper",val:": bool = True"},{name:"keep_torch_compile",val:": bool = True"},{name:"recursive",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.extract_model_from_parallel.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to extract.`,name:"model"},{anchor:"accelerate.utils.extract_model_from_parallel.keep_fp32_wrapper",description:`<strong>keep_fp32_wrapper</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to remove mixed precision hooks from the model.`,name:"keep_fp32_wrapper"},{anchor:"accelerate.utils.extract_model_from_parallel.keep_torch_compile",description:`<strong>keep_torch_compile</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to unwrap compiled model.`,name:"keep_torch_compile"},{anchor:"accelerate.utils.extract_model_from_parallel.recursive",description:`<strong>recursive</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to recursively extract all cases of <code>module.module</code> from <code>model</code> as well as unwrap child sublayers | |
| recursively, not just the top-level distributed containers.`,name:"recursive"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The extracted model.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>torch.nn.Module</code></p> | |
| `}),r(2),o(fe);var be=e(fe,2),_t=a(be);t(_t,{name:"accelerate.utils.get_balanced_memory",anchor:"accelerate.utils.get_balanced_memory",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L938",parameters:[{name:"model",val:": Module"},{name:"max_memory",val:": dict[int | str, int | str] | None = None"},{name:"no_split_module_classes",val:": list[str] | None = None"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"special_dtypes",val:": dict[str, str | torch.device] | None = None"},{name:"low_zero",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.get_balanced_memory.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to analyze.`,name:"model"},{anchor:"accelerate.utils.get_balanced_memory.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. | |
| Example: <code>max_memory={0: "1GB"}</code>.`,name:"max_memory"},{anchor:"accelerate.utils.get_balanced_memory.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of layer class names that should never be split across device (for instance any layer that has a | |
| residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.utils.get_balanced_memory.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.utils.get_balanced_memory.special_dtypes",description:`<strong>special_dtypes</strong> (<code>Dict[str, Union[str, torch.device]]</code>, <em>optional</em>) — | |
| If provided, special dtypes to consider for some specific weights (will override dtype used as default for | |
| all weights).`,name:"special_dtypes"},{anchor:"accelerate.utils.get_balanced_memory.low_zero",description:`<strong>low_zero</strong> (<code>bool</code>, <em>optional</em>) — | |
| Minimizes the number of weights on GPU 0, which is convenient when it’s used for other operations (like the | |
| Transformers generate function).`,name:"low_zero"}]});var Ua=e(_t,4);g(Ua,{children:(c,l)=>{var n=Mt();d(c,n)},$$slots:{default:!0}}),o(be);var ve=e(be,2),za=a(ve);t(za,{name:"accelerate.utils.get_max_layer_size",anchor:"accelerate.utils.get_max_layer_size",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L705",parameters:[{name:"modules",val:": list"},{name:"module_sizes",val:": dict"},{name:"no_split_module_classes",val:": list"}],parametersDescription:[{anchor:"accelerate.utils.get_max_layer_size.modules",description:`<strong>modules</strong> (<code>List[Tuple[str, torch.nn.Module]]</code>) — | |
| The list of named modules where we want to determine the maximum layer size.`,name:"modules"},{anchor:"accelerate.utils.get_max_layer_size.module_sizes",description:`<strong>module_sizes</strong> (<code>Dict[str, int]</code>) — | |
| A dictionary mapping each layer name to its size (as generated by <code>compute_module_sizes</code>).`,name:"module_sizes"},{anchor:"accelerate.utils.get_max_layer_size.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>) — | |
| A list of class names for layers we don’t want to be split.`,name:"no_split_module_classes"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The maximum size of a layer with the list of layer names realizing that maximum size.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Tuple[int, List[str]]</code></p> | |
| `}),r(4),o(ve);var ye=e(ve,2),gt=a(ye);t(gt,{name:"accelerate.infer_auto_device_map",anchor:"accelerate.infer_auto_device_map",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L1319",parameters:[{name:"model",val:": Module"},{name:"max_memory",val:": dict[int | str, int | str] | None = None"},{name:"no_split_module_classes",val:": list[str] | None = None"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"special_dtypes",val:": dict[str, str | torch.dtype] | None = None"},{name:"verbose",val:": bool = False"},{name:"clean_result",val:": bool = True"},{name:"offload_buffers",val:": bool = False"},{name:"fallback_allocation",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.infer_auto_device_map.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to analyze.`,name:"model"},{anchor:"accelerate.infer_auto_device_map.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. | |
| Example: <code>max_memory={0: "1GB"}</code>.`,name:"max_memory"},{anchor:"accelerate.infer_auto_device_map.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of layer class names that should never be split across device (for instance any layer that has a | |
| residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.infer_auto_device_map.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.infer_auto_device_map.special_dtypes",description:`<strong>special_dtypes</strong> (<code>Dict[str, Union[str, torch.device]]</code>, <em>optional</em>) — | |
| If provided, special dtypes to consider for some specific weights (will override dtype used as default for | |
| all weights).`,name:"special_dtypes"},{anchor:"accelerate.infer_auto_device_map.verbose",description:`<strong>verbose</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to provide debugging statements as the function builds the device_map.`,name:"verbose"},{anchor:"accelerate.infer_auto_device_map.clean_result",description:`<strong>clean_result</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Clean the resulting device_map by grouping all submodules that go on the same device together.`,name:"clean_result"},{anchor:"accelerate.infer_auto_device_map.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as | |
| well as the parameters.`,name:"offload_buffers"},{anchor:"accelerate.infer_auto_device_map.fallback_allocation",description:`<strong>fallback_allocation</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| When regular allocation fails, try to allocate a module that fits in the size limit using BFS.`,name:"fallback_allocation"}]});var ja=e(gt,6);g(ja,{children:(c,l)=>{var n=Mt();d(c,n)},$$slots:{default:!0}}),o(ye);var we=e(ye,2),ft=a(we);t(ft,{name:"accelerate.load_checkpoint_in_model",anchor:"accelerate.load_checkpoint_in_model",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L1830",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": str | os.PathLike"},{name:"device_map",val:": dict[str, int | str | torch.device] | None = None"},{name:"offload_folder",val:": str | os.PathLike | None = None"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"offload_state_dict",val:": bool = False"},{name:"offload_buffers",val:": bool = False"},{name:"keep_in_fp32_modules",val:": list[str] | None = None"},{name:"offload_8bit_bnb",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_in_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model in which we want to load a checkpoint.`,name:"model"},{anchor:"accelerate.load_checkpoint_in_model.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder checkpoint to load. It can be: | |
| <ul> | |
| <li>a path to a file containing a whole model state dict</li> | |
| <li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li> | |
| <li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li> | |
| <li>a path to a folder containing a unique pytorch_model.bin or a model.safetensors file.</li> | |
| </ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_in_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_in_model.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) — | |
| If the <code>device_map</code> contains any value <code>"disk"</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_in_model.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_in_model.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if | |
| the weight of the CPU state dict + the biggest shard does not fit.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the buffers in the weights offloaded to disk.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_in_model.keep_in_fp32_modules(List[str],",description:`<strong>keep_in_fp32_modules(<code>List[str]</code>,</strong> <em>optional</em>) — | |
| A list of the modules that we keep in <code>torch.float32</code> dtype.`,name:"keep_in_fp32_modules(List[str],"},{anchor:"accelerate.load_checkpoint_in_model.offload_8bit_bnb",description:`<strong>offload_8bit_bnb</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to enable offload of 8-bit modules on cpu/disk.`,name:"offload_8bit_bnb"},{anchor:"accelerate.load_checkpoint_in_model.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model’s | |
| state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_in_model.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — if this is set to <code>True</code>, all the tensors in the | |
| loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) — when the option is <code>True</code>, a distributed | |
| <code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors | |
| in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) | |
| according to the local shards in the model.`,name:"broadcast_from_rank0"}]});var Ca=e(ft,4);g(Ca,{warning:!0,children:(c,l)=>{var n=Pt();d(c,n)},$$slots:{default:!0}}),o(we);var xe=e(we,2),Aa=a(xe);t(Aa,{name:"accelerate.utils.load_offloaded_weights",anchor:"accelerate.utils.load_offloaded_weights",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L897",parameters:[{name:"model",val:""},{name:"index",val:""},{name:"offload_folder",val:""}],parametersDescription:[{anchor:"accelerate.utils.load_offloaded_weights.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to load the weights into.`,name:"model"},{anchor:"accelerate.utils.load_offloaded_weights.index",description:`<strong>index</strong> (<code>dict</code>) — | |
| A dictionary containing the parameter name and its metadata for each parameter that was offloaded from the | |
| model.`,name:"index"},{anchor:"accelerate.utils.load_offloaded_weights.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code>) — | |
| The folder where the offloaded weights are stored.`,name:"offload_folder"}]}),r(2),o(xe);var ke=e(xe,2),Sa=a(ke);t(Sa,{name:"accelerate.utils.load_state_dict",anchor:"accelerate.utils.load_state_dict",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L1661",parameters:[{name:"checkpoint_file",val:""},{name:"device_map",val:" = None"}],parametersDescription:[{anchor:"accelerate.utils.load_state_dict.checkpoint_file",description:"<strong>checkpoint_file</strong> (<code>str</code>) — The path to the checkpoint to load.",name:"checkpoint_file"},{anchor:"accelerate.utils.load_state_dict.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"}]}),r(2),o(ke);var Te=e(ke,2),Ra=a(Te);t(Ra,{name:"accelerate.utils.offload_state_dict",anchor:"accelerate.utils.offload_state_dict",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/offload.py#L94",parameters:[{name:"save_dir",val:": str | os.PathLike"},{name:"state_dict",val:": dict"}],parametersDescription:[{anchor:"accelerate.utils.offload_state_dict.save_dir",description:`<strong>save_dir</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The directory in which to offload the state dict.`,name:"save_dir"},{anchor:"accelerate.utils.offload_state_dict.state_dict",description:`<strong>state_dict</strong> (<code>Dict[str, torch.Tensor]</code>) — | |
| The dictionary of tensors to offload.`,name:"state_dict"}]}),r(2),o(Te);var Ne=e(Te,2),Wa=a(Ne);t(Wa,{name:"accelerate.utils.retie_parameters",anchor:"accelerate.utils.retie_parameters",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L609",parameters:[{name:"model",val:""},{name:"tied_params",val:""}],parametersDescription:[{anchor:"accelerate.utils.retie_parameters.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model in which to retie parameters.`,name:"model"},{anchor:"accelerate.utils.retie_parameters.tied_params",description:`<strong>tied_params</strong> (<code>List[List[str]]</code>) — | |
| A mapping parameter name to tied parameter name as obtained by <code>find_tied_parameters</code>.`,name:"tied_params"}]}),r(2),o(Ne);var Me=e(Ne,2),Ea=a(Me);t(Ea,{name:"accelerate.utils.set_module_tensor_to_device",anchor:"accelerate.utils.set_module_tensor_to_device",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L210",parameters:[{name:"module",val:": Module"},{name:"tensor_name",val:": str"},{name:"device",val:": int | str | torch.device"},{name:"value",val:": torch.Tensor | None = None"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"fp16_statistics",val:": torch.HalfTensor | None = None"},{name:"tied_params_map",val:": dict[int, dict[torch.device, torch.Tensor]] | None = None"},{name:"non_blocking",val:": bool = False"},{name:"clear_cache",val:": bool = True"}],parametersDescription:[{anchor:"accelerate.utils.set_module_tensor_to_device.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module in which the tensor we want to move lives.`,name:"module"},{anchor:"accelerate.utils.set_module_tensor_to_device.tensor_name",description:`<strong>tensor_name</strong> (<code>str</code>) — | |
| The full name of the parameter/buffer.`,name:"tensor_name"},{anchor:"accelerate.utils.set_module_tensor_to_device.device",description:`<strong>device</strong> (<code>int</code>, <code>str</code> or <code>torch.device</code>) — | |
| The device on which to set the tensor.`,name:"device"},{anchor:"accelerate.utils.set_module_tensor_to_device.value",description:`<strong>value</strong> (<code>torch.Tensor</code>, <em>optional</em>) — | |
| The value of the tensor (useful when going from the meta device to any other device).`,name:"value"},{anchor:"accelerate.utils.set_module_tensor_to_device.dtype",description:`<strong>dtype</strong> (<code>torch.dtype</code>, <em>optional</em>) — | |
| If passed along the value of the parameter will be cast to this <code>dtype</code>. Otherwise, <code>value</code> will be cast to | |
| the dtype of the existing parameter in the model.`,name:"dtype"},{anchor:"accelerate.utils.set_module_tensor_to_device.fp16_statistics",description:`<strong>fp16_statistics</strong> (<code>torch.HalfTensor</code>, <em>optional</em>) — | |
| The list of fp16 statistics to set on the module, used for 8 bit model serialization.`,name:"fp16_statistics"},{anchor:"accelerate.utils.set_module_tensor_to_device.tied_params_map",description:`<strong>tied_params_map</strong> (Dict[int, Dict[torch.device, torch.Tensor]], <em>optional</em>, defaults to <code>None</code>) — | |
| A map of current data pointers to dictionaries of devices to already dispatched tied weights. For a given | |
| execution device, this parameter is useful to reuse the first available pointer of a shared weight on the | |
| device for all others, instead of duplicating memory.`,name:"tied_params_map"},{anchor:"accelerate.utils.set_module_tensor_to_device.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, the device transfer will be asynchronous with respect to the host, if possible.`,name:"non_blocking"},{anchor:"accelerate.utils.set_module_tensor_to_device.clear_cache",description:`<strong>clear_cache</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Whether or not to clear the device cache after setting the tensor on the device.`,name:"clear_cache"}]}),r(2),o(Me);var Pe=e(Me,2),Ia=a(Pe);t(Ia,{name:"accelerate.utils.get_module_children_bottom_up",anchor:"accelerate.utils.get_module_children_bottom_up",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L583",parameters:[{name:"model",val:": Module"},{name:"return_fqns",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.get_module_children_bottom_up.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) — the model to get the children of",name:"model"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>a list of children modules of <code>model</code> in bottom-up order. The last element is the | |
| <code>model</code> itself.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>list[torch.nn.Module]</code></p> | |
| `}),r(2),o(Pe);var bt=e(Pe,2);s(bt,{title:"Parallel",local:"accelerate.utils.extract_model_from_parallel",headingTag:"h2"});var De=e(bt,4),Ga=a(De);t(Ga,{name:"accelerate.utils.extract_model_from_parallel",anchor:"accelerate.utils.extract_model_from_parallel",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L264",parameters:[{name:"model",val:""},{name:"keep_fp32_wrapper",val:": bool = True"},{name:"keep_torch_compile",val:": bool = True"},{name:"recursive",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.extract_model_from_parallel.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to extract.`,name:"model"},{anchor:"accelerate.utils.extract_model_from_parallel.keep_fp32_wrapper",description:`<strong>keep_fp32_wrapper</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to remove mixed precision hooks from the model.`,name:"keep_fp32_wrapper"},{anchor:"accelerate.utils.extract_model_from_parallel.keep_torch_compile",description:`<strong>keep_torch_compile</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether to unwrap compiled model.`,name:"keep_torch_compile"},{anchor:"accelerate.utils.extract_model_from_parallel.recursive",description:`<strong>recursive</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to recursively extract all cases of <code>module.module</code> from <code>model</code> as well as unwrap child sublayers | |
| recursively, not just the top-level distributed containers.`,name:"recursive"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The extracted model.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>torch.nn.Module</code></p> | |
| `}),r(2),o(De);var Le=e(De,2),Ja=a(Le);t(Ja,{name:"accelerate.utils.save",anchor:"accelerate.utils.save",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L402",parameters:[{name:"obj",val:""},{name:"f",val:""},{name:"save_on_each_node",val:": bool = False"},{name:"safe_serialization",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.save.obj",description:`<strong>obj</strong> — | |
| The data to save`,name:"obj"},{anchor:"accelerate.utils.save.f",description:`<strong>f</strong> — | |
| The file (or file-like object) to use to save the data`,name:"f"},{anchor:"accelerate.utils.save.save_on_each_node",description:`<strong>save_on_each_node</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to only save on the global main process`,name:"save_on_each_node"},{anchor:"accelerate.utils.save.safe_serialization",description:`<strong>safe_serialization</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to save <code>obj</code> using <code>safetensors</code> or the traditional PyTorch way (that uses <code>pickle</code>).`,name:"safe_serialization"}]}),r(2),o(Le);var Fe=e(Le,2),qa=a(Fe);t(qa,{name:"accelerate.utils.load",anchor:"accelerate.utils.load",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L452",parameters:[{name:"f",val:""},{name:"map_location",val:" = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.load.f",description:`<strong>f</strong> — | |
| The file (or file-like object) to use to load the data`,name:"f"},{anchor:"accelerate.utils.load.map_location",description:`<strong>map_location</strong> — | |
| a function, <code>torch.device</code>, string or a dict specifying how to remap storage locations`,name:"map_location"},{anchor:"accelerate.utils.load.*kwargs",description:`*<strong>*kwargs</strong> — | |
| Additional keyword arguments to pass to <code>torch.load()</code>.`,name:"*kwargs"}]}),r(4),o(Fe);var Ue=e(Fe,2),vt=a(Ue);t(vt,{name:"accelerate.utils.wait_for_everyone",anchor:"accelerate.utils.wait_for_everyone",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/other.py#L354",parameters:[]});var Ba=e(vt,4);g(Ba,{warning:!0,children:(c,l)=>{var n=lo();d(c,n)},$$slots:{default:!0}}),o(Ue);var yt=e(Ue,2);s(yt,{title:"Random",local:"accelerate.utils.set_seed",headingTag:"h2"});var ze=e(yt,4),Oa=a(ze);t(Oa,{name:"accelerate.utils.set_seed",anchor:"accelerate.utils.set_seed",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/random.py#L40",parameters:[{name:"seed",val:": int"},{name:"device_specific",val:": bool = False"},{name:"deterministic",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.set_seed.seed",description:`<strong>seed</strong> (<code>int</code>) — | |
| The seed to set.`,name:"seed"},{anchor:"accelerate.utils.set_seed.device_specific",description:`<strong>device_specific</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to differ the seed on each device slightly with <code>self.process_index</code>.`,name:"device_specific"},{anchor:"accelerate.utils.set_seed.deterministic",description:`<strong>deterministic</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to use deterministic algorithms where available. Can slow down training.`,name:"deterministic"}]}),r(2),o(ze);var je=e(ze,2),Xa=a(je);t(Xa,{name:"accelerate.utils.synchronize_rng_state",anchor:"accelerate.utils.synchronize_rng_state",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/random.py#L81",parameters:[{name:"rng_type",val:": accelerate.utils.dataclasses.RNGType | None = None"},{name:"generator",val:": torch.Generator | None = None"}]}),o(je);var Ce=e(je,2),Va=a(Ce);t(Va,{name:"accelerate.synchronize_rng_states",anchor:"accelerate.synchronize_rng_states",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/random.py#L163",parameters:[{name:"rng_types",val:": list"},{name:"generator",val:": torch.Generator | None = None"}]}),o(Ce);var wt=e(Ce,2);s(wt,{title:"PyTorch XLA",local:"accelerate.utils.install_xla",headingTag:"h2"});var Ae=e(wt,4),xt=a(Ae);t(xt,{name:"accelerate.utils.install_xla",anchor:"accelerate.utils.install_xla",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/torch_xla.py#L20",parameters:[{name:"upgrade",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.install_xla.upgrade",description:`<strong>upgrade</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to upgrade <code>torch</code> and install the latest <code>torch_xla</code> wheels.`,name:"upgrade"}]});var Za=e(xt,4);u(Za,{anchor:"accelerate.utils.install_xla.example",children:(c,l)=>{var n=h(),i=e(m(n),2);p(i,{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBpbnN0YWxsX3hsYSUwQSUwQWluc3RhbGxfeGxhKHVwZ3JhZGUlM0RUcnVlKQ==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> install_xla | |
| <span class="hljs-meta">>>> </span>install_xla(upgrade=<span class="hljs-literal">True</span>)`,lang:"python",wrap:!1}),d(c,n)},$$slots:{default:!0}}),o(Ae);var kt=e(Ae,2);s(kt,{title:"Loading model weights",local:"accelerate.load_checkpoint_in_model",headingTag:"h2"});var Se=e(kt,4),Tt=a(Se);t(Tt,{name:"accelerate.load_checkpoint_in_model",anchor:"accelerate.load_checkpoint_in_model",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/modeling.py#L1830",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": str | os.PathLike"},{name:"device_map",val:": dict[str, int | str | torch.device] | None = None"},{name:"offload_folder",val:": str | os.PathLike | None = None"},{name:"dtype",val:": str | torch.dtype | None = None"},{name:"offload_state_dict",val:": bool = False"},{name:"offload_buffers",val:": bool = False"},{name:"keep_in_fp32_modules",val:": list[str] | None = None"},{name:"offload_8bit_bnb",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_in_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model in which we want to load a checkpoint.`,name:"model"},{anchor:"accelerate.load_checkpoint_in_model.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder checkpoint to load. It can be: | |
| <ul> | |
| <li>a path to a file containing a whole model state dict</li> | |
| <li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li> | |
| <li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li> | |
| <li>a path to a folder containing a unique pytorch_model.bin or a model.safetensors file.</li> | |
| </ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_in_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_in_model.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) — | |
| If the <code>device_map</code> contains any value <code>"disk"</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_in_model.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_in_model.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if | |
| the weight of the CPU state dict + the biggest shard does not fit.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the buffers in the weights offloaded to disk.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_in_model.keep_in_fp32_modules(List[str],",description:`<strong>keep_in_fp32_modules(<code>List[str]</code>,</strong> <em>optional</em>) — | |
| A list of the modules that we keep in <code>torch.float32</code> dtype.`,name:"keep_in_fp32_modules(List[str],"},{anchor:"accelerate.load_checkpoint_in_model.offload_8bit_bnb",description:`<strong>offload_8bit_bnb</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to enable offload of 8-bit modules on cpu/disk.`,name:"offload_8bit_bnb"},{anchor:"accelerate.load_checkpoint_in_model.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model’s | |
| state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_in_model.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — if this is set to <code>True</code>, all the tensors in the | |
| loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) — when the option is <code>True</code>, a distributed | |
| <code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors | |
| in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) | |
| according to the local shards in the model.`,name:"broadcast_from_rank0"}]});var Ya=e(Tt,4);g(Ya,{warning:!0,children:(c,l)=>{var n=Pt();d(c,n)},$$slots:{default:!0}}),o(Se);var Nt=e(Se,2);s(Nt,{title:"Quantization",local:"accelerate.utils.load_and_quantize_model",headingTag:"h2"});var Re=e(Nt,4),Qa=a(Re);t(Qa,{name:"accelerate.utils.load_and_quantize_model",anchor:"accelerate.utils.load_and_quantize_model",source:"https://github.com/huggingface/accelerate/blob/vr_4477/src/accelerate/utils/bnb.py#L44",parameters:[{name:"model",val:": Module"},{name:"bnb_quantization_config",val:": BnbQuantizationConfig"},{name:"weights_location",val:": str | os.PathLike | None = None"},{name:"device_map",val:": dict[str, int | str | torch.device] | None = None"},{name:"no_split_module_classes",val:": list[str] | None = None"},{name:"max_memory",val:": dict[int | str, int | str] | None = None"},{name:"offload_folder",val:": str | os.PathLike | None = None"},{name:"offload_state_dict",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.utils.load_and_quantize_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| Input model. The model can be already loaded or on the meta device`,name:"model"},{anchor:"accelerate.utils.load_and_quantize_model.bnb_quantization_config",description:`<strong>bnb_quantization_config</strong> (<code>BnbQuantizationConfig</code>) — | |
| The bitsandbytes quantization parameters`,name:"bnb_quantization_config"},{anchor:"accelerate.utils.load_and_quantize_model.weights_location",description:`<strong>weights_location</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder weights_location to load. It can be: | |
| <ul> | |
| <li>a path to a file containing a whole model state dict</li> | |
| <li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li> | |
| <li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li> | |
| <li>a path to a folder containing a unique pytorch_model.bin file.</li> | |
| </ul>`,name:"weights_location"},{anchor:"accelerate.utils.load_and_quantize_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"},{anchor:"accelerate.utils.load_and_quantize_model.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of layer class names that should never be split across device (for instance any layer that has a | |
| residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.utils.load_and_quantize_model.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset.`,name:"max_memory"},{anchor:"accelerate.utils.load_and_quantize_model.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) — | |
| If the <code>device_map</code> contains any value <code>"disk"</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.utils.load_and_quantize_model.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if | |
| the weight of the CPU state dict + the biggest shard does not fit.`,name:"offload_state_dict"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The quantized model</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>torch.nn.Module</code></p> | |
| `}),r(2),o(Re);var Ha=e(Re,2);to(Ha,{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/utilities.md"}),r(2),d(Dt,We),no()}export{go as component}; | |
Xet Storage Details
- Size:
- 155 kB
- Xet hash:
- 03406e7664b27aa981ed635ab42e7757497295cc0fe551a80acf0e0a16bcff31
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.