mpeg-diffusion / third_party /musubi_tuner /modules /custom_offloading_utils.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
6febba8 verified
Raw History Blame Contribute Delete
58.2 kB
from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass import gc import time from typing import Optional import torch import torch.nn as nn # Keep these functions here for portability, and private to avoid confusion with the ones in device_utils.py def _clean_memory_on_device(device: torch.device): r""" Clean memory on the specified device, will be called from training scripts. """ gc.collect() # device may "cuda" or "cuda:0", so we need to check the type of device if device.type == "cuda": torch.cuda.empty_cache() if device.type == "xpu": torch.xpu.empty_cache() if device.type == "mps": torch.mps.empty_cache() def _synchronize_device(device: torch.device): if device.type == "cuda": torch.cuda.synchronize() elif device.type == "xpu": torch.xpu.synchronize() elif device.type == "mps": torch.mps.synchronize() def swap_weight_devices_no_cuda(device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module): """ not tested """ assert layer_to_cpu.__class__ == layer_to_cuda.__class__ weight_swap_jobs = [] for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()): if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None: weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)) # device to cpu for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: module_to_cpu.weight.data = cuda_data_view.data.to("cpu", non_blocking=True) _synchronize_device(device) # cpu to device for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs: cuda_data_view.copy_(module_to_cuda.weight.data, non_blocking=True) module_to_cuda.weight.data = cuda_data_view _synchronize_device(device) def weighs_to_device(layer: nn.Module, device: torch.device): for module in layer.modules(): if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"): module.weight.data = module.weight.data.to(device, non_blocking=device.type != "cpu") def default_swap_tensor_selector(block: nn.Module) -> list[tuple[nn.Module, str]]: """Default streaming selection rule: the ``weight`` of every ``*Linear`` module. Same rule as ``ModelOffloader.swap_weight_devices_cuda``. An offloader can be given a different selector to also stream tensors that a quantization scheme hangs off the same modules (per-group scales etc.); each entry is ``(module, attribute_name)`` where the attribute is a parameter or a registered buffer of the module. """ jobs = [] for _, module in block.named_modules(): if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"): jobs.append((module, "weight")) return jobs def attach_forward_streaming_hooks(offloader, blocks: list[nn.Module]) -> list: """Drive a block-swap offloader from forward hooks. For models whose forward loop does not call the offloader itself (e.g. a Hugging Face text encoder): each block waits for its own weights before running and releases/prefetches after it finishes. Forward-only offloaders handle the pass boundary (wrap-around preload) internally. Hooks must be attached before any per-call hooks so the prefetch of the last block runs first. Returns the hook handles so callers can detach them. """ handles = [] for index, block in enumerate(blocks): def _wait_hook(module, args, _index=index): del module, args offloader.wait_for_block(_index) def _submit_hook(module, args, output, _index=index): del module, args, output offloader.submit_move_blocks_forward(blocks, _index) handles.append(block.register_forward_pre_hook(_wait_hook)) handles.append(block.register_forward_hook(_submit_hook)) return handles @dataclass class BlockSwapConfig: """ Construction policy for a block-swap offloader, assembled once by the training/inference script and passed through each architecture's ``enable_block_swap`` to ``create_offloader``. It holds everything the offloader constructor needs that is *not* architecture-specific. The architecture-specific arguments (the block-type label, the block list, and its counts) are supplied by ``enable_block_swap`` per block list. Adding a new offloader knob (or a whole new offloader type) means adding a field here and a branch in ``create_offloader`` -- the per-architecture ``enable_block_swap`` signatures stay ``(blocks_to_swap, config)`` and never change. """ device: torch.device supports_backward: bool use_pinned_memory: bool = False h2d_only: bool = False # frozen-base (LoRA / LoHa / LoKr) only: H2D-only streaming, no device->host copy ring_size: int = 2 # (h2d_only) number of GPU ring buffers for streamed blocks; 2 = double buffering debug: bool = False @classmethod def from_args(cls, args, device: torch.device, supports_backward: bool) -> "BlockSwapConfig": """Build from a parsed-args namespace, tolerating scripts whose parser lacks the optional knobs.""" h2d_only = getattr(args, "block_swap_h2d_only", False) # H2D-only streams frozen weights into a reused GPU ring buffer and loads them with an in-place copy_, # which bumps the autograd version of the saved weight view. Gradient checkpointing re-reads the weight # at recompute time (a short, safe window), so the version stays consistent; without it the backward # pass fails with an opaque "modified by an inplace operation ... version N expected M" error that does # not point at the offloader. Fail early with an actionable message instead. (Inference / forward-only # has no backward, so it is unaffected.) if h2d_only and supports_backward and not getattr(args, "gradient_checkpointing", False): raise ValueError( "--block_swap_h2d_only requires --gradient_checkpointing for training. H2D-only block swap streams" " frozen weights through a reused GPU ring buffer, which advances the autograd version of weights" " saved for backward; gradient checkpointing re-reads them at recompute time and avoids this." " / --block_swap_h2d_only は学習時に --gradient_checkpointing が必須です(リングバッファの上書きで" "backward 用に保存された重みの version が進むため。gradient checkpointing は再計算時に読み直すので回避できます)。" ) ring_size = getattr(args, "block_swap_ring_size", 2) if ring_size < 1: raise ValueError("--block_swap_ring_size must be >= 1") return cls( device=device, supports_backward=supports_backward, use_pinned_memory=getattr(args, "use_pinned_memory_for_block_swap", False), h2d_only=h2d_only, ring_size=ring_size, ) def create_offloader(block_type: str, blocks: list[nn.Module], num_blocks: int, blocks_to_swap: int, config: BlockSwapConfig): """ Create the block-swap offloader for one block list. This is the single place that selects the offloader implementation from ``config``; ``enable_block_swap`` only supplies the per-block-list arguments, so a new offloader type plugs in here without touching any architecture. """ if config.h2d_only: # H2D-only streaming for frozen-base (LoRA) training: keep a CPU master, copy Host->Device only. return LoRAStreamOffloader( block_type, blocks, num_blocks, blocks_to_swap, config.supports_backward, config.device, ring_size=config.ring_size, use_pinned_memory=config.use_pinned_memory, debug=config.debug, ) return ModelOffloader( block_type, blocks, num_blocks, blocks_to_swap, config.supports_backward, config.device, config.use_pinned_memory, debug=config.debug, ) class Offloader: """ common offloading class """ def __init__( self, block_type: str, num_blocks: int, blocks_to_swap: int, device: torch.device, use_pinned_memory: bool = False, debug: bool = False, ): self.block_type = block_type self.num_blocks = num_blocks self.blocks_to_swap = blocks_to_swap self.device = device self.use_pinned_memory = use_pinned_memory # check if debug is enabled from os environment variable if not debug: import os debug = os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG", "0") == "1" self.debug = debug self.debug_block_count = 0 self.thread_pool = ThreadPoolExecutor(max_workers=1) self.futures = {} self.cuda_available = device.type == "cuda" self.stream = torch.cuda.Stream(device=device) if self.cuda_available else None # Staging buffers for cuda offloading without large pinned memory. These are pinned memory buffers to speed up the transfer between CPU and GPU # We create one staging buffer per transfer direction (A: GPU to CPU, B: CPU to GPU) self.staging_buffer_a = None self.staging_buffer_b = None # Pinned buffer for cuda offloading with pinned memory. We need only one pinned buffer per layer transfer self.pinned_buffer = None def swap_weight_devices_cuda(self, device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module): assert layer_to_cpu.__class__ == layer_to_cuda.__class__ debug_print = False if self.debug: debug_print = self.debug_block_count % 10 == 0 self.debug_block_count += 1 class Timer: def __init__(self, enabled=False): self.enabled = enabled self.totals = defaultdict(float) self.start_time = time.perf_counter() @contextmanager def section(self, name): if not self.enabled: yield return t0 = time.perf_counter() try: yield finally: self.totals[name] += time.perf_counter() - t0 T = Timer(enabled=debug_print) weight_swap_jobs = [] # This is not working for all cases (e.g. SD3), so we need to find the corresponding modules. kept here for reference: # for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()): # print(module_to_cpu.__class__, module_to_cuda.__class__) # if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None: # weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data)) with T.section("find modules"): modules_to_cpu = {k: v for k, v in layer_to_cpu.named_modules()} for module_to_cuda_name, module_to_cuda in layer_to_cuda.named_modules(): if ( hasattr(module_to_cuda, "weight") and module_to_cuda.weight is not None and module_to_cuda.__class__.__name__.endswith("Linear") ): module_to_cpu = modules_to_cpu.get(module_to_cuda_name, None) if module_to_cpu is not None and module_to_cpu.weight.shape == module_to_cuda.weight.shape: weight_swap_jobs.append( (module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data) ) else: if module_to_cuda.weight.data.device.type != device.type: module_to_cuda.weight.data = module_to_cuda.weight.data.to(device) with T.section("synchronize before swap"): torch.cuda.current_stream().synchronize() # this prevents the illegal loss value by ensuring offloading layer's calculation is done if not self.use_pinned_memory: # Minimize using pinned memory for lower shared GPU RAM usage stream = self.stream with torch.cuda.stream(stream): if self.staging_buffer_a is None: # Create staging buffer as pinned memory (as shared GPU ram). We specify device for correct pinning on multi-GPU systems self.staging_buffer_a = [ torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device) for _, _, cuda_data_view, _ in weight_swap_jobs ] self.staging_buffer_b = [ torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device) for _, _, cuda_data_view, _ in weight_swap_jobs ] # Copy weights to staging buffers and record events event_b = None for sbuf_a, sbuf_b, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip( self.staging_buffer_a, self.staging_buffer_b, weight_swap_jobs ): # CUDA to staging buffer A, non-blocking copy event_a = torch.cuda.Event() with T.section("cuda to staging A"): sbuf_a.copy_(cuda_data_view.data, non_blocking=True) event_a.record(stream) # Wait for staging buffer B to be ready if event_b is not None: with T.section("wait staging B"): event_b.synchronize() # synchronize is needed to wait CPU process. wait_event does not work here because it waits on GPU side only # CPU to staging buffer B, CPU to pinned CPU, synchronous copy. Can overlap with CUDA to staging buffer A with T.section("cpu to staging B"): # Making this multithreaded does not help, and 'non_blocking=True' does not help either. sbuf_b.copy_(module_to_cuda.weight.data) # BOTTLENECK # Wait for staging buffer A to be ready, and CUDA data view can be reused with T.section("wait staging A"): event_a.synchronize() # Staging buffer B to CUDA, non-blocking copy. event_b = torch.cuda.Event() with T.section("staging B to CUDA"): cuda_data_view.copy_(sbuf_b, non_blocking=True) event_b.record(stream) # Staging buffer A to CPU, synchronous copy. Can overlap with staging buffer B to CUDA with T.section("staging A to CPU"): cpu_data_view.copy_(sbuf_a) # BOTTLENECK for sbuf_a, sbuf_b, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip( self.staging_buffer_a, self.staging_buffer_b, weight_swap_jobs ): # Update references module_to_cuda.weight.data = cuda_data_view module_to_cpu.weight.data = cpu_data_view sync_event = event_b # final sync event for CPU to CUDA copy else: # Use pinned memory for faster transfer between CPU and GPU, but it requires more memory if self.pinned_buffer is None: with torch.cuda.stream(self.stream): # Create pinned buffer as pinned memory (as shared GPU ram). We specify device for correct pinning on multi-GPU systems self.pinned_buffer = [ torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device) for _, _, cuda_data_view, _ in weight_swap_jobs ] self.stream.synchronize() released_pinned_buffer = [] events = [torch.cuda.Event() for _ in weight_swap_jobs] # Waiting events for GPU to CPU non-blocking copy # Copy weights to CPU for event, module_pin_buf, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip( events, self.pinned_buffer, weight_swap_jobs ): # CUDA to CPU, non-blocking copy with torch.cuda.stream(self.stream): with T.section("cuda to cpu"): module_pin_buf.copy_(cuda_data_view, non_blocking=True) event.record(self.stream) # CPU to CUDA for event, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip(events, weight_swap_jobs): with torch.cuda.stream(self.stream): # Wait for cuda_data_view to be ready with T.section("wait cpu"): self.stream.wait_event(event) # CPU to CUDA, non-blocking copy with T.section("cpu to cuda"): cuda_data_view.copy_(cpu_data_view, non_blocking=True) # Update references for module_pin_buf, (module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view) in zip( self.pinned_buffer, weight_swap_jobs ): module_to_cuda.weight.data = cuda_data_view module_to_cpu.weight.data = module_pin_buf released_pinned_buffer.append(cpu_data_view) # CPU data view can be reused as pinned buffer # Reuse released pinned buffers if not released_pinned_buffer[0].is_pinned(): # In first time, we need to create pinned buffers because offloaded weights are not pinned yet with torch.cuda.stream(self.stream): released_pinned_buffer = [ torch.empty_like(cuda_data_view, device="cpu").pin_memory(device=device) for _, _, cuda_data_view, _ in weight_swap_jobs ] self.pinned_buffer = released_pinned_buffer sync_event = self.stream.record_event() if debug_print: print(f"[{self.block_type}] Weight swap timing at {self.debug_block_count - 1}:") for name, total in T.totals.items(): print(f" {name}: {total * 1000:.2f}ms") print( f"Overall time: {(time.perf_counter() - T.start_time) * 1000:.2f}ms, total time in sections: {sum(T.totals.values()) * 1000:.2f}ms" ) # print( # f"[{self.block_type}] Swapped weights in {time.perf_counter() - start_time:.2f}s. Count of modules swapped: {len(weight_swap_jobs)}" # ) return sync_event def swap_weight_devices(self, block_to_cpu: nn.Module, block_to_cuda: nn.Module): if self.cuda_available: sync_event = self.swap_weight_devices_cuda(self.device, block_to_cpu, block_to_cuda) else: swap_weight_devices_no_cuda(self.device, block_to_cpu, block_to_cuda) sync_event = None return sync_event def _submit_move_blocks(self, blocks, block_idx_to_cpu, block_idx_to_cuda): def move_blocks(bidx_to_cpu, block_to_cpu, bidx_to_cuda, block_to_cuda): if self.debug: start_time = time.perf_counter() print( f"[{self.block_type}] Move block {bidx_to_cpu} to CPU and block {bidx_to_cuda} to {'CUDA' if self.cuda_available else 'device'}" ) dev = self.device.index if self.device.index is not None else torch.cuda.current_device() torch.cuda.set_device(dev) sync_event = self.swap_weight_devices(block_to_cpu, block_to_cuda) if self.debug: print( f"[{self.block_type}] Moved blocks {bidx_to_cpu} to CPU and {bidx_to_cuda} to {'CUDA' if self.cuda_available else 'device'} in {time.perf_counter() - start_time:.2f}s" ) return bidx_to_cpu, bidx_to_cuda, sync_event block_to_cpu = blocks[block_idx_to_cpu] block_to_cuda = blocks[block_idx_to_cuda] self.futures[block_idx_to_cuda] = self.thread_pool.submit( move_blocks, block_idx_to_cpu, block_to_cpu, block_idx_to_cuda, block_to_cuda ) def _wait_blocks_move(self, block_idx): if block_idx not in self.futures: return if self.debug: print(f"[{self.block_type}] Wait for block {block_idx}") start_time = time.perf_counter() future = self.futures.pop(block_idx) _, bidx_to_cuda, sync_event = future.result() assert block_idx == bidx_to_cuda, f"Block index mismatch: {block_idx} != {bidx_to_cuda}" if self.cuda_available and sync_event is not None: # this does not wait CPU side, so the log below should be immediate when pinned memory is used torch.cuda.current_stream().wait_event(sync_event) if self.debug: print(f"[{self.block_type}] Waited for block {block_idx}: {time.perf_counter() - start_time:.2f}s") class ModelOffloader(Offloader): """ supports forward offloading """ def __init__( self, block_type: str, blocks: list[nn.Module], num_blocks: int, blocks_to_swap: int, supports_backward: bool, device: torch.device, use_pinned_memory: bool = False, debug: bool = False, ): super().__init__(block_type, num_blocks, blocks_to_swap, device, use_pinned_memory, debug) self.supports_backward = supports_backward self.forward_only = not supports_backward # forward only offloading: can be changed to True for inference if self.supports_backward: # register backward hooks self.remove_handles = [] for i, block in enumerate(blocks): hook = self.create_backward_hook(blocks, i) if hook is not None: handle = block.register_full_backward_hook(hook) self.remove_handles.append(handle) def set_forward_only(self, forward_only: bool): # switching must wait for all pending transfers for block_idx in list(self.futures.keys()): self._wait_blocks_move(block_idx) self.forward_only = forward_only def __del__(self): if self.supports_backward: for handle in self.remove_handles: handle.remove() def create_backward_hook(self, blocks: list[nn.Module], block_index: int) -> Optional[callable]: # -1 for 0-based index num_blocks_propagated = self.num_blocks - block_index - 1 swapping = num_blocks_propagated > 0 and num_blocks_propagated <= self.blocks_to_swap waiting = block_index > 0 and block_index <= self.blocks_to_swap if not swapping and not waiting: return None # create hook block_idx_to_cpu = self.num_blocks - num_blocks_propagated block_idx_to_cuda = self.blocks_to_swap - num_blocks_propagated block_idx_to_wait = block_index - 1 def backward_hook(module, grad_input, grad_output): if self.debug: print(f"Backward hook for block {block_index}") if swapping: self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda) if waiting: self._wait_blocks_move(block_idx_to_wait) return None return backward_hook def prepare_block_devices_before_forward(self, blocks: list[nn.Module]): if self.blocks_to_swap is None or self.blocks_to_swap == 0: return if self.debug: print(f"[{self.block_type}] Prepare block devices before forward") for b in blocks[0 : self.num_blocks - self.blocks_to_swap]: b.to(self.device) weighs_to_device(b, self.device) # make sure weights are on device cpu_device = torch.device("cpu") for b in blocks[self.num_blocks - self.blocks_to_swap :]: b.to(self.device) # move block to device first. this makes sure that buffers (non weights) are on the device weighs_to_device(b, cpu_device) # make sure weights are on cpu _synchronize_device(self.device) _clean_memory_on_device(self.device) def wait_for_block(self, block_idx: int): if self.blocks_to_swap is None or self.blocks_to_swap == 0: return self._wait_blocks_move(block_idx) def submit_move_blocks_forward(self, blocks: list[nn.Module], block_idx: int): # check if blocks_to_swap is enabled if self.blocks_to_swap is None or self.blocks_to_swap == 0: return if not self.forward_only: # if backward is enabled, we do not swap blocks in forward pass more than blocks_to_swap, because it should be on GPU if block_idx >= self.blocks_to_swap: return block_idx_to_cpu = block_idx block_idx_to_cuda = self.num_blocks - self.blocks_to_swap + block_idx block_idx_to_cuda = block_idx_to_cuda % self.num_blocks # this does nothing for backward offloading self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda) return # We use two strategies here for forward-only offloading: # 1. If blocks_to_swap is less than half of num_blocks, we swap the num_blocks blocks without wrapping around. # This reduces the number of swaps, so it is especially useful for small blocks_to_swap or lightweight models like Qwen-Image # 2. If blocks_to_swap is more than half of num_blocks, we swap the blocks with wrapping around. # This is the common strategy used in most offloading implementations. It transfers all blocks in a wrapping manner. # This is useful for large blocks_to_swap or heavyweight models like Wan/HunyuanVideo, where the transfer time is less significant compared to computation time. # current block to swap out (to CPU) block_idx_to_cpu = block_idx if self.blocks_to_swap < (self.num_blocks // 2): # strategy 1: no wrap around # If the current block is in the middle blocks that are not swapped, do nothing if self.blocks_to_swap <= block_idx < self.num_blocks - self.blocks_to_swap: return if block_idx < self.blocks_to_swap: # move the next block to cuda block_idx_to_cuda = (self.num_blocks - self.blocks_to_swap + block_idx) % self.num_blocks else: # move the previous block to cuda block_idx_to_cuda = block_idx - (self.num_blocks - self.blocks_to_swap) else: # strategy 2: with wrap around block_idx_to_cuda = self.num_blocks - self.blocks_to_swap + block_idx block_idx_to_cuda = block_idx_to_cuda % self.num_blocks # this works for forward-only offloading self._submit_move_blocks(blocks, block_idx_to_cpu, block_idx_to_cuda) class _DirectCopier: """ Transfer engine for ``LoRAStreamOffloader`` when the masters are *pinned*: copies a flat pinned host buffer straight into a flat device buffer on a private copy stream (async Host->Device only). This is the seam that isolates all CUDA stream/event primitives from the residency logic. The residency side only ever calls ``submit`` / ``wait``; ``_StagedCopier`` (pageable master + worker thread) and a future non-CUDA device shim plug in behind the same interface without the residency code changing. Keep every ``torch.cuda.*`` transfer primitive inside copiers like this one. """ def __init__(self, device: torch.device, debug: bool = False): self.device = device self.debug = debug self.copy_stream = torch.cuda.Stream(device=device) self._events = {} # key -> cuda.Event recording H2D completion self._xfer = {} # key -> (ev_a, ev_b) transfer-timing events (debug only) def submit(self, key, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event=None): """Begin the H2D copy of ``src_flat`` into ``dst_flat``, gated behind ``gate_event`` (compute done with dst).""" ev_a = ev_b = None with torch.cuda.stream(self.copy_stream): if gate_event is not None: self.copy_stream.wait_event(gate_event) # do not overwrite a buffer still used by compute if self.debug: ev_a = torch.cuda.Event(enable_timing=True) ev_b = torch.cuda.Event(enable_timing=True) ev_a.record(self.copy_stream) dst_flat.copy_(src_flat, non_blocking=True) # src is pinned -> async H2D if self.debug: ev_b.record(self.copy_stream) self._xfer[key] = (ev_a, ev_b) self._events[key] = self.copy_stream.record_event() def wait(self, key): """Return the H2D-completion event for ``key`` (None if nothing is in flight for it).""" return self._events.get(key) def pop_xfer_timing(self, key): """Pop the (start, end) transfer-timing events recorded for ``key`` (debug only).""" return self._xfer.pop(key, None) def sync(self): self.copy_stream.synchronize() def reset(self): self._events.clear() self._xfer.clear() class _StagedCopier: """ Transfer engine for ``LoRAStreamOffloader`` when the masters are kept in *pageable* host memory: each H2D is staged through a small pool of pinned buffers filled by a background worker thread. Same ``submit`` / ``wait`` interface as ``_DirectCopier``, so the residency code is identical. Why: pinning every streaming master (what ``_DirectCopier`` needs) can cost tens of GB of non-pageable "shared GPU memory" (notably on Windows). Here the masters stay pageable and only ``num_staging`` pinned buffers exist. A pageable->device copy is, inside the CUDA driver, already a synchronous serial pageable->pinned->device transfer; doing it ourselves on a worker thread lets the pageable->pinned memcpy overlap the previous block's pinned->device DMA and -- crucially -- keeps it off the main thread so compute kernel launches are never blocked. That is the whole speed difference versus the old synchronous pageable path this replaces. Hazards handled: - a staging buffer is reused only after the H2D that consumed it has completed (one consume-event per buffer; the worker waits on it before overwriting); - ``wait`` blocks the caller until the worker has actually enqueued the H2D and recorded its completion event, so the residency layer can ``current_stream().wait_event`` it as usual. """ def __init__(self, device: torch.device, num_staging: int = 2, debug: bool = False): self.device = device self.num_staging = max(1, num_staging) self.debug = debug self.copy_stream = torch.cuda.Stream(device=device) self._pool = ThreadPoolExecutor(max_workers=1) self._futures = {} # key -> Future -> (load_event, (ev_a, ev_b)) self._xfer = {} # key -> (ev_a, ev_b) transfer-timing events (debug only) self._staging = None # list[pinned uint8 tensor] (lazily sized to the flat block) self._staging_free = None # list[cuda.Event | None]: the H2D that last consumed each buffer self._rr = 0 # round-robin staging index (touched only on the worker thread) def _ensure_staging(self, nbytes: int): if self._staging is None: self._staging = [torch.empty(nbytes, dtype=torch.uint8).pin_memory(device=self.device) for _ in range(self.num_staging)] self._staging_free = [None] * self.num_staging def _run(self, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event): # runs on the worker thread: pageable->pinned memcpy, then enqueue the async pinned->device H2D idx = self._rr self._rr = (idx + 1) % self.num_staging staging = self._staging[idx] free_ev = self._staging_free[idx] if free_ev is not None: free_ev.synchronize() # CPU(worker) waits until the prior H2D released this buffer -- off main thread staging.copy_(src_flat) # pageable -> pinned, plain CPU memcpy ev_a = ev_b = None with torch.cuda.stream(self.copy_stream): if gate_event is not None: self.copy_stream.wait_event(gate_event) # do not overwrite a dst still used by compute if self.debug: ev_a = torch.cuda.Event(enable_timing=True) ev_b = torch.cuda.Event(enable_timing=True) ev_a.record(self.copy_stream) dst_flat.copy_(staging, non_blocking=True) # pinned -> device, async if self.debug: ev_b.record(self.copy_stream) done = self.copy_stream.record_event() # H2D complete == this staging buffer is free again self._staging_free[idx] = done return done, (ev_a, ev_b) def submit(self, key, dst_flat: torch.Tensor, src_flat: torch.Tensor, gate_event=None): self._ensure_staging(src_flat.numel()) # flat uint8 -> numel == byte size self._futures[key] = self._pool.submit(self._run, dst_flat, src_flat, gate_event) def wait(self, key): fut = self._futures.get(key) if fut is None: return None load_event, xfer = fut.result() # block until the worker enqueued the H2D (tiny if it ran ahead) if self.debug and xfer[0] is not None: self._xfer[key] = xfer return load_event def pop_xfer_timing(self, key): return self._xfer.pop(key, None) def _flush(self): for fut in list(self._futures.values()): fut.result() def sync(self): self._flush() self.copy_stream.synchronize() def reset(self): self._flush() self.copy_stream.synchronize() # ensure all in-flight H2D are done before clearing staging state self._futures.clear() self._xfer.clear() if self._staging is not None: self._staging_free = [None] * self.num_staging def __del__(self): pool = getattr(self, "_pool", None) if pool is not None: pool.shutdown(wait=False) class LoRAStreamOffloader: """ H2D-only offloader for training where the base weights are frozen (e.g. LoRA / LoHa / LoKr). The classic block swap (``ModelOffloader``) *exchanges* a block: it copies one block back to the CPU (Device->Host) while copying the next one in (Host->Device). When the base weights never change, the CPU already holds an identical copy, so the D2H half is pure overhead. This offloader keeps a permanent (pinned) master copy of every streamed weight on the CPU and only ever copies Host->Device, removing the D2H transfer and the CPU-side staging memcpy entirely. Layout (N = num_blocks, S = blocks_to_swap, B = ring_size): - ``N - S`` blocks stay resident on the GPU permanently. - ``S`` "streaming" blocks are placed at even intervals; their ``Linear`` weights live on the CPU (pinned) as masters and are streamed into a small ring of ``B`` GPU buffers (default 2: one being computed on, one being prefetched). Non-Linear params / buffers of streaming blocks stay resident. Mechanics: - Because the streamed weights are frozen (no ``.grad`` to preserve), we swap the whole ``module.weight`` *reference* between its CPU master Parameter and a preallocated GPU ring Parameter, instead of the ``.data`` storage trick used by ``ModelOffloader``. No per-step ``cudaMalloc``. - What is streamed is decided by ``swap_tensor_selector`` (default: every ``*Linear`` ``weight``). A selector may return several tensors per module -- e.g. a quantized weight together with its scale buffers -- as ``(module, attribute_name)`` pairs; parameters are rebound as ``nn.Parameter`` ring views, registered buffers as plain tensor views. - All swap weights of a block are packed into a single contiguous (byte) buffer -- one pinned CPU buffer per streaming block and one GPU buffer per ring slot -- and the individual weights are dtype/shape views into it. Loading a block is therefore a single H2D ``copy_`` instead of one per Linear, which removes the per-tensor launch/driver overhead on PCIe/CPU-bound systems. - Eviction is free: repoint ``module.weight`` back to the CPU master (no transfer). A ring slot is overwritten only when its block is consumed, so the last ``B`` streamed blocks of a pass stay resident. On the reverse pass they are exactly the first blocks needed, so the forward<->backward turn-around needs no reload. - Direction is implicit in the callback: ``submit_move_blocks_forward`` prefetches the block ``B`` streaming-slots *ahead* (larger index); the backward hook prefetches ``B`` slots *behind* (smaller index). No explicit scheduler / no ``prepare_*_before_backward``. - ``wait_for_block`` self-heals: if a needed streaming block is not resident in its slot (cold start, or an inference pass boundary), it is loaded on demand. """ def __init__( self, block_type: str, blocks: list[nn.Module], num_blocks: int, blocks_to_swap: int, supports_backward: bool, device: torch.device, ring_size: int = 2, use_pinned_memory: bool = True, debug: bool = False, swap_tensor_selector=None, ): self.block_type = block_type self._blocks = blocks self.num_blocks = num_blocks # N self.blocks_to_swap = blocks_to_swap # requested S self.device = device self.use_pinned_memory = use_pinned_memory self.supports_backward = supports_backward self.forward_only = not supports_backward self.swap_tensor_selector = swap_tensor_selector if swap_tensor_selector is not None else default_swap_tensor_selector import os if not debug: debug = os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG", "0") == "1" self.debug = debug self.debug_interval = int(os.getenv("MUSUBI_TUNER_OFFLOADER_DEBUG_INTERVAL", "10")) # steps between prints assert device.type == "cuda", "LoRAStreamOffloader currently supports CUDA only" # ---- streaming placement: S evenly spaced block indices (midpoint formula -> distinct for S <= N) ---- stream_idx = sorted({((2 * i + 1) * num_blocks) // (2 * blocks_to_swap) for i in range(blocks_to_swap)}) self.stream_idx = stream_idx self.S = len(stream_idx) # actual streaming count (>=1; dedup is a no-op unless S is close to N) self.rank = {b: k for k, b in enumerate(stream_idx)} # block_idx -> position in stream_idx self.is_stream = [b in self.rank for b in range(num_blocks)] self.B = min(ring_size, self.S) # clamp ring to number of streaming blocks # ---- finetuning guard: H2D-only must never lose weight updates ---- self._job_cache = {} # block_idx -> [(module, attr_name, is_param) per swap tensor] for b in stream_idx: for m, name, is_param in self._jobs(b): assert not (is_param and getattr(m, name).requires_grad), ( "LoRAStreamOffloader requires frozen base weights (LoRA / no full fine-tune). " f"Found a trainable swap tensor {name} in block {b}." ) # ---- transfer engine: owns the copy stream and all H2D primitives ---- # pinned masters -> direct async H2D; pageable masters -> stage through a worker thread + pinned pool # (low "shared GPU memory" footprint, e.g. on Windows). Both expose the same submit/wait interface. if use_pinned_memory: self.copier = _DirectCopier(device, debug=self.debug) else: self.copier = _StagedCopier(device, num_staging=self.B, debug=self.debug) # ---- runtime state (GPU buffers allocated lazily in prepare_block_devices_before_forward) ---- self.cpu_master = {} # block_idx -> [CPU (pinned) Parameter / buffer tensor per swap tensor] (views into cpu_flat) self.cpu_flat = {} # block_idx -> flat (pinned) uint8 CPU tensor backing the masters self.ring_param = None # [slot] -> [GPU nn.Parameter / buffer tensor per swap tensor] (views into ring_flat) self.ring_flat = None # [slot] -> flat uint8 GPU tensor backing the ring params self._layout = None # ([byte offset per swap tensor], total bytes) shared by all streaming blocks self.in_slot = [None] * self.B # slot -> block_idx currently bound to this slot (or None) self.free_event = [None] * self.B # slot -> cuda.Event recording when compute finished using the slot self._wait_ctx = "fwd" # context tag for wait/load timing ("fwd" while in the forward loop, "bwd" in hooks) if self.debug: self._dbg_reset() # ---- backward hooks: only where there is work (prefetch this block, or wait the previous one) ---- if supports_backward: self.remove_handles = [] for i, block in enumerate(blocks): hook = self._create_backward_hook(i) if hook is not None: self.remove_handles.append(block.register_full_backward_hook(hook)) print( f"LoRAStreamOffloader[{block_type}]: H2D-only block swap. " f"{self.S} streaming / {num_blocks} blocks, ring={self.B}, pinned={use_pinned_memory}. " f"streaming indices: {stream_idx}" ) # ------------------------------------------------------------------ helpers def _jobs(self, block_idx: int) -> list[tuple[nn.Module, str, bool]]: """Swap jobs of a block: ``(module, attr_name, is_param)`` per streamed tensor, cached.""" cached = self._job_cache.get(block_idx) if cached is None: cached = [] for m, name in self.swap_tensor_selector(self._blocks[block_idx]): if name in m._parameters: is_param = True elif name in m._buffers: is_param = False else: raise ValueError( f"swap tensor selector returned {name!r}, which is neither a parameter nor a" f" registered buffer of {m.__class__.__name__} (block {block_idx})" ) cached.append((m, name, is_param)) self._job_cache[block_idx] = cached return cached def _bind(self, block_idx: int, tensors: list[torch.Tensor]): # parameters arrive as nn.Parameter, buffers as plain tensors; setattr keeps them in the # module's _parameters / _buffers dicts respectively (buffer names are already registered) for (m, name, _is_param), t in zip(self._jobs(block_idx), tensors): setattr(m, name, t) @staticmethod def _compute_layout(weights: list[torch.Tensor]) -> tuple[list[int], int]: """Byte offsets (aligned so ``view(dtype)`` is valid for any element size) and total size of the flat buffer.""" align = 256 offsets = [] total = 0 for w in weights: total = (total + align - 1) // align * align offsets.append(total) total += w.numel() * w.element_size() return offsets, total @staticmethod def _flat_views(flat: torch.Tensor, weights: list[torch.Tensor], layout: tuple[list[int], int]) -> list[torch.Tensor]: """dtype/shape views into a flat uint8 buffer, one per swap tensor, following ``layout``.""" offsets, _ = layout return [flat[off : off + w.numel() * w.element_size()].view(w.dtype).view(w.shape) for off, w in zip(offsets, weights)] def _load(self, rank: int, slot: int, ctx: str = "fwd"): """Stream the streaming block at ``rank`` into ring ``slot`` (H2D only), repointing weights.""" blk = self.stream_idx[rank] # already resident (e.g. ring survived a re-prepare): the slot content is still identical to the # immutable CPU master, so just rebind the weights to the ring -- no transfer needed if self.in_slot[slot] == blk: self._bind(blk, self.ring_param[slot]) return # evict the slot's current owner: repoint its weights back to the CPU master (no transfer) prev = self.in_slot[slot] if prev is not None: self._bind(prev, self.cpu_master[prev]) # single coalesced H2D copy of the whole block, gated behind the slot's free_event (compute done # with it). The ring params are views into ring_flat, so the copy bumps their autograd version # counter: safe under gradient checkpointing (weights are re-read at recompute time, and a slot is # only overwritten after its free_event); without checkpointing autograd raises a version error # instead of silently using stale-saved weights -- which the old per-tensor `.data.copy_` hid. self.copier.submit(blk, self.ring_flat[slot], self.cpu_flat[blk], self.free_event[slot]) self._bind(blk, self.ring_param[slot]) # reference swap (no .data trick, no cudaMalloc) self.in_slot[slot] = blk if self.debug: c = self._cur[ctx] c["loads"] += 1 xfer = self.copier.pop_xfer_timing(blk) if xfer is not None: c["xfer_ev"].append(xfer) # ------------------------------------------------------------------ public interface (mirrors ModelOffloader) def set_forward_only(self, forward_only: bool): self.copier.sync() self.forward_only = forward_only def __del__(self): if getattr(self, "supports_backward", False): for handle in getattr(self, "remove_handles", []): handle.remove() def prepare_block_devices_before_forward(self, blocks: list[nn.Module]): if self.S == 0: return if self.debug: print(f"[{self.block_type}] Prepare block devices before forward (H2D-only)") first_time = not self.cpu_master # device placement is only needed on the very first call cpu_device = torch.device("cpu") for i, b in enumerate(blocks): if not self.is_stream[i]: if first_time: b.to(self.device) # resident block: buffers + Linear weights on GPU permanently weighs_to_device(b, self.device) continue if not first_time: # re-prepare (e.g. around in-training sampling): the non-swap parts are already on the device # and the CPU masters have stayed on the CPU throughout, so just repoint weights to the masters. # IMPORTANT: do NOT call b.to(device) here -- it would drag the CPU masters onto the GPU and # they would never be released (every swap block would end up resident). self._bind(i, self.cpu_master[i]) continue jobs = self._jobs(i) tensors = [getattr(m, name).data for m, name, _ in jobs] if self._layout is None: self._layout = self._compute_layout(tensors) # first streaming block defines the shared layout flat = torch.empty(self._layout[1], dtype=torch.uint8, device=cpu_device) if self.use_pinned_memory: flat = flat.pin_memory(device=self.device) views = self._flat_views(flat, tensors, self._layout) master = [] if all(t.device.type == "cpu" for t in tensors): # CPU-direct: build the flat masters host-to-host, detach the swap tensors from the # module so the block's device move cannot drag them, then restore them as the masters. # Avoids the H2D+D2H round trip of every streamed tensor that the general path below # pays when the model starts on the CPU. for (m, name, is_param), t, view in zip(jobs, tensors, views): view.copy_(t) # host-to-host memcpy into the flat master placeholder = torch.empty(0, dtype=t.dtype, device=cpu_device) if is_param: m._parameters[name].data = placeholder else: m._buffers[name] = placeholder b.to(self.device) # non-swap params/buffers to the device; the placeholders are empty for (m, name, is_param), view in zip(jobs, views): if is_param: p = m._parameters[name] p.data = view master.append(p) # keep the original Parameter object as the persistent master else: m._buffers[name] = view master.append(view) else: # swap tensors already live on an accelerator: move the whole block to the device # (buffers, norms, bias), then pull the swap tensors back into the flat master (one-time D2H) b.to(self.device) for (m, name, is_param), view in zip(jobs, views): t = getattr(m, name) view.copy_(t.data) if is_param: t.data = view master.append(t) # keep the original Parameter object as the persistent master else: m._buffers[name] = view master.append(view) self.cpu_flat[i] = flat self.cpu_master[i] = master # validate homogeneous streaming blocks (shared ring template) template = self.cpu_master[self.stream_idx[0]] for b in self.stream_idx[1:]: assert len(self.cpu_master[b]) == len(template), f"block {b} has a different number of swap tensors" for p, t in zip(self.cpu_master[b], template): assert p.data.shape == t.data.shape and p.data.dtype == t.data.dtype, ( f"block {b} swap-tensor shape/dtype differs from the streaming template" ) # preallocate the GPU ring (once); copies happen into these flat buffers, never reallocated if self.ring_param is None: template_jobs = self._jobs(self.stream_idx[0]) template_weights = [p.data for p in template] self.ring_flat = [torch.empty(self._layout[1], dtype=torch.uint8, device=self.device) for _ in range(self.B)] self.ring_param = [ [ nn.Parameter(view, requires_grad=False) if is_param else view for (_m, _name, is_param), view in zip(template_jobs, self._flat_views(flat, template_weights, self._layout)) ] for flat in self.ring_flat ] # reset transfer state and preload the first B streaming blocks. in_slot deliberately survives: # after a completed backward pass the ring already holds ranks 0..B-1, so these loads become # no-op rebinds (see _load) instead of redundant cold-start transfers. self.free_event = [None] * self.B self.copier.reset() for k in range(self.B): self._load(k, k) _synchronize_device(self.device) _clean_memory_on_device(self.device) def wait_for_block(self, block_idx: int): if self.S == 0 or not self.is_stream[block_idx]: return ctx = self._wait_ctx if self.debug and ctx == "fwd" and block_idx == self.stream_idx[0]: self._dbg_boundary() # a forward pass is starting -> finalize the previous step's stats j = self.rank[block_idx] slot = j % self.B if self.in_slot[slot] != block_idx: # self-healing: not resident (cold start / inference pass boundary / same-direction restart) if self.debug: self._cur[ctx]["self_heal"] += 1 self._load(j, slot, ctx) ev = self.copier.wait(block_idx) if ev is None: return if not self.debug: torch.cuda.current_stream().wait_event(ev) # compute waits for H2D (GPU-side, does not block CPU) return # debug: measure the actual GPU stall (idle time the compute stream spends on wait_event) s = torch.cuda.current_stream() c = self._cur[ctx] c["waits"] += 1 if not ev.query(): c["not_ready"] += 1 # H2D not finished when compute reached it -> a real stall (cheap, no sync) ev_a = torch.cuda.Event(enable_timing=True) ev_b = torch.cuda.Event(enable_timing=True) ev_a.record(s) s.wait_event(ev) ev_b.record(s) c["stall_ev"].append((ev_a, ev_b)) # staged copier records its transfer-timing events on the worker thread, so they only become # available once wait() has joined it; the direct copier already popped them in _load (returns None) xfer = self.copier.pop_xfer_timing(block_idx) if xfer is not None: c["xfer_ev"].append(xfer) def submit_move_blocks_forward(self, blocks: list[nn.Module], block_idx: int): # consumed block_idx in the forward pass: free its slot and prefetch the block B streaming-slots ahead if self.S == 0 or not self.is_stream[block_idx]: return j = self.rank[block_idx] self.free_event[j % self.B] = torch.cuda.current_stream().record_event() if j + self.B < self.S: self._load(j + self.B, (j + self.B) % self.B, "fwd") # same slot as rank j; evicts rank j elif self.forward_only and j == self.S - 1 and self.S > self.B: # inference: no backward pass will consume the retained tail, and the next forward pass starts # from rank 0 again. Wrap around: all tail slots are free once the last streaming block is # consumed, so preload the first B ranks now. The H2D overlaps with the remaining (resident) # blocks and the output layers, instead of stalling at the start of the next pass (self-heal). for k in range(self.B): self._load(k, k, "fwd") def _create_backward_hook(self, block_index: int): prefetch = self.is_stream[block_index] wait_prev = block_index - 1 >= 0 and self.is_stream[block_index - 1] if not prefetch and not wait_prev: return None def backward_hook(module, grad_input, grad_output): if prefetch: # consumed block_index in backward: free its slot and prefetch B streaming-slots behind j = self.rank[block_index] self.free_event[j % self.B] = torch.cuda.current_stream().record_event() if j - self.B >= 0: self._load(j - self.B, (j - self.B) % self.B, "bwd") # same slot as rank j; evicts rank j if wait_prev: self._wait_ctx = "bwd" self.wait_for_block(block_index - 1) # ensure the next block to run is ready self._wait_ctx = "fwd" return None return backward_hook # ------------------------------------------------------------------ debug timing def _dbg_reset(self): self._dbg_empty = lambda: {"stall_ev": [], "xfer_ev": [], "waits": 0, "not_ready": 0, "self_heal": 0, "loads": 0} self._cur = {"fwd": self._dbg_empty(), "bwd": self._dbg_empty()} self._roll = {"fwd": defaultdict(float), "bwd": defaultdict(float)} self._step = 0 def _dbg_boundary(self): # finalize the events accumulated for the step that just ended, then print every debug_interval steps if not any(self._cur[c]["waits"] or self._cur[c]["loads"] for c in ("fwd", "bwd")): return # very first forward: nothing finished yet torch.cuda.synchronize() # ensure timing events are complete (debug only) for ctx in ("fwd", "bwd"): c, r = self._cur[ctx], self._roll[ctx] r["stall_ms"] += sum(a.elapsed_time(b) for a, b in c["stall_ev"]) r["xfer_ms"] += sum(a.elapsed_time(b) for a, b in c["xfer_ev"]) for k in ("waits", "not_ready", "self_heal", "loads"): r[k] += c[k] self._cur[ctx] = self._dbg_empty() self._step += 1 if self._step % self.debug_interval == 0: self._dbg_print() def _dbg_print(self): n = self.debug_interval print(f"[{self.block_type}] H2D-only timing (avg over {n} steps):") for ctx in ("fwd", "bwd"): r = self._roll[ctx] print( f" {ctx}: stall {r['stall_ms'] / n:6.2f} ms/step, H2D {r['xfer_ms'] / n:6.2f} ms/step" f" | waits {r['waits'] / n:.1f}, not-ready {r['not_ready'] / n:.1f}," f" self-heal {r['self_heal'] / n:.1f}, loads {r['loads'] / n:.1f}" ) self._roll[ctx] = defaultdict(float)
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass
import gc
import time
from typing import Optional
import torch
import torch.nn as nn
# Keep these functions here for portability, and private to avoid confusion with the ones in device_utils.py
def _clean_memory_on_device(device: torch.device):
r"""
Clean memory on the specified device, will be called from training scripts.
"""
gc.collect()
# device may "cuda" or "cuda:0", so we need to check the type of device
if device.type == "cuda":
torch.cuda.empty_cache()
if device.type == "xpu":
torch.xpu.empty_cache()
if device.type == "mps":
torch.mps.empty_cache()
def _synchronize_device(device: torch.device):
if device.type == "cuda":
torch.cuda.synchronize()
elif device.type == "xpu":
torch.xpu.synchronize()
elif device.type == "mps":
torch.mps.synchronize()
def swap_weight_devices_no_cuda(device: torch.device, layer_to_cpu: nn.Module, layer_to_cuda: nn.Module):
"""
not tested
"""
assert layer_to_cpu.__class__ == layer_to_cuda.__class__
weight_swap_jobs = []
for module_to_cpu, module_to_cuda in zip(layer_to_cpu.modules(), layer_to_cuda.modules()):
if hasattr(module_to_cpu, "weight") and module_to_cpu.weight is not None:
weight_swap_jobs.append((module_to_cpu, module_to_cuda, module_to_cpu.weight.data, module_to_cuda.weight.data))
# device to cpu
for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs:
module_to_cpu.weight.data = cuda_data_view.data.to("cpu", non_blocking=True)
_synchronize_device(device)
# cpu to device
for module_to_cpu, module_to_cuda, cuda_data_view, cpu_data_view in weight_swap_jobs:
cuda_data_view.copy_(module_to_cuda.weight.data, non_blocking=True)
module_to_cuda.weight.data = cuda_data_view
_synchronize_device(device)
def weighs_to_device(layer: nn.Module, device: torch.device):
for module in layer.modules():
if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"):
module.weight.data = module.weight.data.to(device, non_blocking=device.type != "cpu")
def default_swap_tensor_selector(block: nn.Module) -> list[tuple[nn.Module, str]]:
"""Default streaming selection rule: the ``weight`` of every ``*Linear`` module.
Same rule as ``ModelOffloader.swap_weight_devices_cuda``. An offloader can be given a
different selector to also stream tensors that a quantization scheme hangs off the same
modules (per-group scales etc.); each entry is ``(module, attribute_name)`` where the
attribute is a parameter or a registered buffer of the module.
"""
jobs = []
for _, module in block.named_modules():
if hasattr(module, "weight") and module.weight is not None and module.__class__.__name__.endswith("Linear"):
jobs.append((module, "weight"))
return jobs
def attach_forward_streaming_hooks(offloader, blocks: list[nn.Module]) -> list:
"""Drive a block-swap offloader from forward hooks.
For models whose forward loop does not call the offloader itself (e.g. a Hugging Face
text encoder): each block waits for its own weights before running and releases/prefetches
after it finishes. Forward-only offloaders handle the pass boundary (wrap-around preload)
internally. Hooks must be attached before any per-call hooks so the prefetch of the last
block runs first. Returns the hook handles so callers can detach them.
"""
handles = []
for index, block in enumerate(blocks):
def _wait_hook(module, args, _index=index):
del module, args
offloader.wait_for_block(_index)
def _submit_hook(module, args, output, _index=index):
del module, args, output