tarekziade HF Staff
RFC: device_map form implemented, measured warm start; transformers.patch at PR 1a1652cb5b
65d3c91 verified Download transformers.patch from tarekziade/moe-expert-streaming: direct link, hf CLI and curl.
- Browser
- Download file 104 kB
-
https://huggingface.co/spaces/tarekziade/moe-expert-streaming/resolve/main/transformers.patch
- Command line
-
hf download hf://spaces/tarekziade/moe-expert-streaming/transformers.patch
-
curl -L -o transformers.patch https://huggingface.co/spaces/tarekziade/moe-expert-streaming/resolve/main/transformers.patch
104 kB
diff --git a/src/transformers/integrations/experts_offload.py b/src/transformers/integrations/experts_offload.py
new file mode 100644
index 0000000000..7a7686bdc0
--- /dev/null
+++ b/src/transformers/integrations/experts_offload.py
@@ -0,0 +1,1341 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Stream MoE experts from disk through a shared device cache. Inference only, on one device."""
+
+import heapq
+import json
+import math
+import os
+import re
+import threading
+from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+
+from ..utils import logging
+from .moe import ExpertsInterface
+
+
+try:
+ import fcntl
+except ImportError: # Windows
+ fcntl = None
+
+logger = logging.get_logger(__name__)
+
+_DTYPES = {"F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16}
+_PACKED_DTYPES = {**_DTYPES, "U32": torch.uint32, "U8": torch.uint8}
+_QUANTIZED_FORMAT = "mlx-affine-expert-blob"
+_MLX_QUANTIZATION_KERNEL = ("kernels-community/mlx-quantization-metal-kernels", 2)
+_COUNTER_LOCK = threading.Lock()
+
+
+def _read_exact(path, offset, size, bypass_page_cache, what, out=None, fd=None):
+ """Read a byte range into ``out`` (or a new buffer), optionally bypassing or dropping the OS page cache.
+
+ ``fd`` is an open descriptor of ``path``; positional reads are thread-safe."""
+ # TODO(portability): implement Windows cache bypass; report unsupported OS/filesystem combinations.
+ data = bytearray(size) if out is None else out
+ own = fd is None
+ if own:
+ fd = os.open(path, os.O_RDONLY)
+ if bypass_page_cache and hasattr(fcntl, "F_NOCACHE"):
+ fcntl.fcntl(fd, fcntl.F_NOCACHE, 1)
+ try:
+ view = memoryview(data).cast("B")
+ done = 0
+ while done < size:
+ count = os.preadv(fd, [view[done:size]], offset + done)
+ if not count:
+ raise OSError(f"Short checkpoint read: {path}, {what}")
+ done += count
+ # TODO(portability): distinguish best-effort cache eviction from uncached reads.
+ if bypass_page_cache and hasattr(os, "posix_fadvise"):
+ os.posix_fadvise(fd, offset, size, os.POSIX_FADV_DONTNEED)
+ finally:
+ if own:
+ os.close(fd)
+ return data
+
+
+class SafetensorsReader:
+ """Validate safetensors headers and read tensors or contiguous expert slices without persistent mmap."""
+
+ def __init__(self, directory, bypass_page_cache=False, packed=False):
+ directory = Path(directory)
+ self.bypass_page_cache = bypass_page_cache
+ self.dtypes = _PACKED_DTYPES if packed else _DTYPES
+ index = directory / "model.safetensors.index.json"
+ if index.exists():
+ weight_map = json.loads(index.read_text())["weight_map"]
+ filenames = sorted(set(weight_map.values()))
+ else:
+ weight_map = None
+ filenames = ["model.safetensors"]
+ self.tensors = {}
+ self.read_bytes = 0
+ self.read_calls = 0
+ self._fds = {}
+ for filename in filenames:
+ path = (directory / filename).resolve()
+ if not path.is_file():
+ raise FileNotFoundError(f"Missing checkpoint shard: {path}")
+ # Validate offsets, sizes and overlaps before reading the header.
+ with safe_open(path, framework="pt", device="cpu") as handle:
+ keys = set(handle.keys())
+ with path.open("rb") as handle:
+ header_size = int.from_bytes(handle.read(8), "little")
+ header = json.loads(handle.read(header_size))
+ for name in keys:
+ info = header[name]
+ if name in self.tensors:
+ raise ValueError(f"Duplicate checkpoint tensor: {name}")
+ self.tensors[name] = (path, 8 + header_size + info["data_offsets"][0], info)
+ if weight_map is not None and weight_map.get(name) != filename:
+ raise ValueError(f"Checkpoint index disagrees with shard for {name}")
+ if weight_map is not None and set(weight_map) != set(self.tensors):
+ raise ValueError("Checkpoint index contains missing tensors")
+
+ def shape(self, name):
+ return tuple(self.tensors[name][2]["shape"])
+
+ def read(self, name, expert=None, out=None):
+ """Read a tensor, or one expert slice of it, into ``out`` (a CPU uint8 tensor of its size) when given."""
+ path, offset, info = self.tensors[name]
+ shape = tuple(info["shape"])
+ if info["dtype"] not in self.dtypes:
+ raise ValueError(f"Only F32/F16/BF16 (and packed) tensors can be read: {name} is {info['dtype']}")
+ dtype = self.dtypes[info["dtype"]]
+ item_size = torch.empty((), dtype=dtype).element_size()
+ if expert is not None:
+ if len(shape) < 2 or not 0 <= expert < shape[0]:
+ raise ValueError(f"Invalid expert slice {expert} for {name}: {shape}")
+ shape = shape[1:]
+ offset += expert * math.prod(shape) * item_size
+ size = math.prod(shape) * item_size
+ if out is not None:
+ if out.dtype != torch.uint8 or out.numel() != size or out.device.type != "cpu":
+ raise ValueError(f"`out` must be a CPU uint8 tensor of {size} bytes")
+ _read_exact(path, offset, size, self.bypass_page_cache, f"tensor {name}", out.numpy(), self._fd(path))
+ data = out
+ else:
+ data = torch.frombuffer(
+ _read_exact(path, offset, size, self.bypass_page_cache, f"tensor {name}", fd=self._fd(path)),
+ dtype=torch.uint8,
+ )
+ with _COUNTER_LOCK:
+ self.read_bytes += size
+ self.read_calls += 1
+ return data.view(dtype).reshape(shape)
+
+ def _fd(self, path):
+ """One read-only descriptor per shard, opened on first use and shared by the I/O threads."""
+ fd = self._fds.get(path)
+ if fd is None:
+ with _COUNTER_LOCK:
+ fd = self._fds.get(path)
+ if fd is None:
+ fd = os.open(path, os.O_RDONLY)
+ if self.bypass_page_cache and hasattr(fcntl, "F_NOCACHE"):
+ fcntl.fcntl(fd, fcntl.F_NOCACHE, 1)
+ self._fds[path] = fd
+ return fd
+
+ def __del__(self):
+ for fd in getattr(self, "_fds", {}).values():
+ try:
+ os.close(fd)
+ except OSError:
+ pass
+
+ def read_expert(self, prefix, expert):
+ if f"{prefix}.gate_up_proj" in self.tensors:
+ return self.read(f"{prefix}.gate_up_proj", expert), self.read(f"{prefix}.down_proj", expert)
+ gate_up = torch.cat([self.read(f"{prefix}.{expert}.{p}_proj.weight") for p in ("gate", "up")])
+ return gate_up, self.read(f"{prefix}.{expert}.down_proj.weight")
+
+
+def _layout_bytes(layout):
+ return sum(math.prod(shape) * dtype.itemsize for shape, dtype in layout)
+
+
+class _Pool:
+ """Device slots shared by the MoE layers whose experts have one layout; LRU (or LFU) within the pool."""
+
+ def __init__(self, layout, num_slots, packed):
+ self.layout = layout # [(shape, dtype)] of one expert's tensors, in storage order
+ self.slot_bytes = _layout_bytes(layout)
+ self.num_slots = num_slots
+ self.packed = packed
+ self.slot_buffer = self.banks = None # see `materialize`
+ self.free_slots = list(range(num_slots - 1, -1, -1))
+ self.cache = OrderedDict() # key -> slot, least recently used first
+ self.heap = [] # LFU: (count, sequence, key); stale entries are discarded lazily
+ self.num_experts = 0 # over the pool's layers
+
+ def materialize(self, device):
+ if self.banks is not None:
+ return
+ if self.packed:
+ # One row of bytes per slot, viewed as the expert's tensors: one copy per install.
+ self.slot_buffer = torch.empty((self.num_slots, self.slot_bytes), device=device, dtype=torch.uint8)
+ self.banks = list(_split_expert(self.slot_buffer, self.layout))
+ else:
+ self.banks = [
+ torch.empty((self.num_slots, *shape), device=device, dtype=dtype) for shape, dtype in self.layout
+ ]
+
+
+class ExpertStore:
+ """Device pools of expert slots with LRU or decaying LFU admission, for one inference request at a time.
+
+ Layers whose experts share a layout share a pool; with several layouts (mixed quantization, an external expert
+ source), the budget gives every layer the same number of slots. Staging and uncached experts can exceed the budget.
+
+ Experts come from the checkpoint (floating point), from `quantized_experts` (MLX affine bytes), or from an
+ ``expert_source``: an object with ``layout(prefix)`` (``[(shape, dtype)]`` of one expert's tensors, stored
+ contiguously in that order), ``read(prefix, expert, out)`` (fill ``out``, a CPU uint8 tensor of the expert's size)
+ and ``read_bytes`` / ``read_calls`` counters; optionally ``prefix(module_name)`` (its key for an experts module)
+ and ``pool_key(prefix)`` (layers with equal keys share a pool; defaults to the layout)."""
+
+ def __init__(
+ self,
+ reader,
+ cache_bytes,
+ device,
+ dtype,
+ policy="lru",
+ decay_every=4096,
+ io_threads=0,
+ quantized=None,
+ source=None,
+ ):
+ if cache_bytes < 0:
+ raise ValueError("cache_bytes must be nonnegative")
+ if policy not in ("lru", "lfu"):
+ raise ValueError(f"Unknown cache policy {policy!r}; use 'lru' or 'lfu'")
+ if source is not None and quantized is not None:
+ raise ValueError("Use either quantized_experts or expert_source, not both")
+ self.policy = policy
+ self.decay_every = decay_every
+ self.counts = {}
+ self.sequence = 0
+ self.lookups = 0
+ self.not_admitted = 0
+ self.reader = reader
+ self.source = source
+ self.cache_bytes = cache_bytes
+ self.device = torch.device(device)
+ self.dtype = dtype
+ self.resident_bytes = 0
+ self.peak_cache_bytes = 0
+ self.hits = self.misses = self.evictions = 0
+ self.quantized = None
+ if quantized is not None:
+ info = json.loads((Path(quantized) / "quantization.json").read_text(encoding="utf-8"))
+ if info.get("format") != _QUANTIZED_FORMAT:
+ raise ValueError(f"Unsupported quantized experts format {info.get('format')!r}")
+ self.quantized = (
+ SafetensorsReader(quantized, reader.bypass_page_cache, packed=True),
+ info["bits"],
+ info["group_size"],
+ )
+ self.gather_qmm = _load_gather_qmm(self.device)
+ # Layers register first (`register`), then `allocate` sizes the pools from the whole set of layouts.
+ self.layouts = {} # prefix -> (pool key, layout, num_experts)
+ self.pools = []
+ self.pool_of = {}
+ # expert -> slot per layer (the pool's num_slots when absent), kept on the host (installs and evictions are host
+ # bookkeeping) and copied to the device only when a device-side lookup needs it.
+ self.slot_tables = {}
+ self.host_slot_tables = {}
+ self.stale_slot_tables = set()
+ self.total_experts = 0
+ self.compute = "grouped_mm"
+ self.expert_compute = None # see `experts_offload_forward`
+ self.uses = {} # (prefix, expert) -> lookups, see `routing_profile`
+ self.preload_profile = None # see `preload_from_profile`
+ self.executor = ThreadPoolExecutor(io_threads, thread_name_prefix="experts-offload-io") if io_threads else None
+ self.pending = {}
+ self.window = 2 * io_threads
+ self.prefetched = self.prefetch_used = self.installed_ahead = 0
+ # Long prefill calls (`cold_prefill_routes`): hits do not refresh, and the experts installed for a layer move to
+ # the cold end of its pool once that layer's call is done, so one long prompt does not flush the decode set.
+ self.cold = False
+ self.cold_installed = {}
+ # Retain host buffers until the next routing copy synchronizes the device.
+ self.inflight = []
+ # Recycled host buffers for expert bytes, all of the largest expert size, see `_host_buffer`.
+ self.free_host_buffers = []
+ self.host_buffer_bytes = 0
+ self.commit = False # see `_commit_mps`
+ # Optional host tier (expert bytes, one layout): a pool of expert rows in host memory between storage and the
+ # device cache, see `allocate_host_cache`. `host_busy` holds the rows handed out and not yet released.
+ self.host_pool = None
+ self.host_slots = 0
+ self.host_cache = OrderedDict()
+ self.host_free = []
+ self.host_busy = set()
+ self.host_lock = threading.Lock()
+ self.host_hits = self.host_misses = 0
+
+ def register(self, prefix, module):
+ """Validate one experts module's tensors and record its layout; returns the checkpoint keys it replaces."""
+ if self.pools:
+ raise ValueError("Register every experts module before `allocate`")
+ shapes = {
+ "gate_up_proj": (module.num_experts, 2 * module.intermediate_dim, module.hidden_dim),
+ "down_proj": (module.num_experts, module.hidden_dim, module.intermediate_dim),
+ }
+ per_expert = [
+ f"{prefix}.{expert}.{projection}.weight"
+ for expert in range(module.num_experts)
+ for projection in ("gate_proj", "up_proj", "down_proj")
+ ]
+ if self.source is not None or (
+ self.quantized is not None and f"{prefix}.experts" in self.quantized[0].tensors
+ ):
+ # Experts come from elsewhere: skip whatever the checkpoint holds for them.
+ candidates = [f"{prefix}.{suffix}" for suffix in shapes] + per_expert
+ names = {name: None for name in candidates if name in self.reader.tensors}
+ elif f"{prefix}.gate_up_proj" in self.reader.tensors:
+ names = {f"{prefix}.{suffix}": shape for suffix, shape in shapes.items()}
+ else:
+ names = {}
+ for expert in range(module.num_experts):
+ for projection in ("gate_proj", "up_proj", "down_proj"):
+ shape = (module.intermediate_dim, module.hidden_dim)
+ if projection == "down_proj":
+ shape = shape[::-1]
+ names[f"{prefix}.{expert}.{projection}.weight"] = shape
+ for name, shape in names.items():
+ if shape is None:
+ continue
+ if name not in self.reader.tensors or self.reader.shape(name) != shape:
+ raise ValueError(f"Missing or incompatible expert tensor {name}; expected {shape}")
+ if self.source is not None:
+ layout = [(tuple(shape), dtype) for shape, dtype in self.source.layout(prefix)]
+ key = self.source.pool_key(prefix) if hasattr(self.source, "pool_key") else None
+ elif self.quantized is not None:
+ self._check_quantized(prefix, module)
+ layout, key = self._quantized_shapes(module), None
+ else:
+ layout = [
+ ((2 * module.intermediate_dim, module.hidden_dim), self.dtype),
+ ((module.hidden_dim, module.intermediate_dim), self.dtype),
+ ]
+ key = None
+ self.layouts[prefix] = (key if key is not None else tuple(layout), layout, module.num_experts)
+ self.total_experts += module.num_experts
+ return set(names)
+
+ def allocate(self, materialize=True):
+ """Size the device pools: one per layout, each layer getting the same number of slots. Their memory is taken
+ now, or with ``materialize=False`` at the first lookup (after loading, so the loading peak excludes them)."""
+ groups = {}
+ for prefix, (key, layout, _) in self.layouts.items():
+ groups.setdefault(key, (layout, []))[1].append(prefix)
+ layer_bytes = [_layout_bytes(layout) for _, layout, _ in self.layouts.values()]
+ if not groups:
+ raise ValueError("No experts module registered")
+ if len(groups) == 1:
+ sizes = [self.cache_bytes // layer_bytes[0]]
+ else:
+ per_layer = self.cache_bytes // sum(layer_bytes)
+ sizes = [per_layer * len(prefixes) for _, prefixes in groups.values()]
+ packed = self.quantized is not None
+ for (layout, prefixes), num_slots in zip(groups.values(), sizes):
+ pool = _Pool(layout, num_slots, packed)
+ self.pools.append(pool)
+ for prefix in prefixes:
+ self.pool_of[prefix] = pool
+ experts = self.layouts[prefix][2]
+ pool.num_experts += experts
+ self.slot_tables[prefix] = torch.full((experts,), num_slots, device=self.device)
+ self.host_slot_tables[prefix] = torch.full((experts,), num_slots)
+ self.host_buffer_bytes = max(pool.slot_bytes for pool in self.pools)
+ self.materialized = False
+ if materialize:
+ self.materialize()
+
+ def materialize(self):
+ """Take the pools' device memory (once), then fill it from ``preload_profile`` when set."""
+ if not self.materialized:
+ for pool in self.pools:
+ pool.materialize(self.device)
+ self.materialized = True
+ if self.preload_profile is not None:
+ self.preload_from_profile(self.preload_profile)
+
+ def routing_profile(self):
+ """Lookups per expert since loading, ``{prefix: {expert: count}}``: what ``preload_profile`` takes."""
+ profile = {}
+ for (prefix, expert), count in self.uses.items():
+ profile.setdefault(prefix, {})[expert] = count
+ return profile
+
+ def preload_from_profile(self, profile):
+ """Fill every layer's share of its pool with its most used experts in ``profile`` (``{prefix: {expert:
+ count}}``, or a path to it as JSON), least used first so the most used end up hottest. Reads in parallel."""
+ if not isinstance(profile, dict):
+ profile = json.loads(Path(profile).read_text(encoding="utf-8"))
+ layers = {}
+ for prefix in self.layouts:
+ layers.setdefault(id(self.pool_of[prefix]), []).append(prefix)
+ cold, self.cold = self.cold, False # the first call may be a cold prefill: the preload is not part of it
+ for prefix in self.layouts:
+ pool = self.pool_of[prefix]
+ counts = {int(expert): count for expert, count in profile.get(prefix, {}).items()}
+ share = pool.num_slots // len(layers[id(pool)])
+ ranked = sorted(counts, key=counts.get, reverse=True)[:share]
+ self.prefetch(prefix, ranked)
+ for expert in reversed(ranked):
+ self._install((prefix, expert))
+ self.synchronize() # releases this layer's host buffers before the next layer's reads
+ self.cold = cold
+
+ # Single-pool views, kept for callers that predate size classes.
+ @property
+ def num_slots(self):
+ return sum(pool.num_slots for pool in self.pools)
+
+ @property
+ def slot_bytes(self):
+ if len(self.pools) != 1:
+ raise ValueError("slot_bytes is defined for a single pool; see `pools`")
+ return self.pools[0].slot_bytes
+
+ @property
+ def cache(self):
+ """Every resident key -> slot (read-only view over the pools)."""
+ return {key: slot for pool in self.pools for key, slot in pool.cache.items()}
+
+ @property
+ def gate_up_pool(self):
+ return self.pools[0].banks[0] if len(self.pools) == 1 and self.quantized is None else None
+
+ @property
+ def down_pool(self):
+ return self.pools[0].banks[1] if len(self.pools) == 1 and self.quantized is None else None
+
+ def _quantized_shapes(self, module):
+ """Per-expert shapes and dtypes of the quantized tensors, in pool order."""
+ _, bits, group_size = self.quantized
+ shapes = []
+ for rows, columns in (
+ (2 * module.intermediate_dim, module.hidden_dim),
+ (module.hidden_dim, module.intermediate_dim),
+ ):
+ if columns % group_size or (columns * bits) % 32:
+ raise ValueError(f"Input size {columns} does not fit {bits}-bit groups of {group_size}")
+ shapes += [
+ ((rows, columns * bits // 32), torch.uint32),
+ ((rows, columns // group_size), self.dtype),
+ ((rows, columns // group_size), self.dtype),
+ ]
+ return shapes
+
+ def _check_quantized(self, prefix, module):
+ reader, name = self.quantized[0], f"{prefix}.experts"
+ size = sum(math.prod(shape) * dtype.itemsize for shape, dtype in self._quantized_shapes(module))
+ if name not in reader.tensors or reader.shape(name) != (module.num_experts, size):
+ raise ValueError(
+ f"Missing or incompatible quantized experts {name}; expected {module.num_experts} x {size} bytes "
+ "(quantized from a checkpoint in the execution dtype)"
+ )
+
+ def all_resident(self):
+ """Whether every expert is cached, allowing routing to stay on the device."""
+ return sum(len(pool.cache) for pool in self.pools) == self.total_experts
+
+ def _count(self, key):
+ self.lookups += 1
+ if self.policy != "lfu":
+ return
+ if self.decay_every and self.lookups % self.decay_every == 0:
+ self.counts = {k: v / 2 for k, v in self.counts.items() if v >= 0.25}
+ for pool in self.pools:
+ pool.heap = []
+ for resident in pool.cache:
+ self._push(resident)
+ self.counts[key] = self.counts.get(key, 0) + 1
+ if key in self.pool_of[key[0]].cache:
+ self._push(key)
+
+ def _push(self, key):
+ self.sequence += 1
+ heapq.heappush(self.pool_of[key[0]].heap, (self.counts.get(key, 0), self.sequence, key))
+
+ def _least_used(self, pool):
+ while pool.heap:
+ count, _, key = pool.heap[0]
+ if key in pool.cache and count == self.counts.get(key, 0):
+ return key, count
+ heapq.heappop(pool.heap)
+ return None, None
+
+ def _evict(self, pool, victim):
+ slot = pool.cache.pop(victim)
+ self.resident_bytes -= pool.slot_bytes
+ self.evictions += 1
+ pool.free_slots.append(slot)
+ self.host_slot_tables[victim[0]][victim[1]] = pool.num_slots
+ self.stale_slot_tables.add(victim[0])
+
+ def _make_room(self, key, pinned=()):
+ """Evict unpinned experts of the key's pool until a slot is free; return False if admission is declined."""
+ pool = self.pool_of[key[0]]
+ if not pool.num_slots:
+ return False
+ if self.policy == "lfu":
+ while not pool.free_slots:
+ victim, count = self._least_used(pool)
+ if victim is None or victim in pinned or self.counts.get(key, 0) <= count:
+ self.not_admitted += 1
+ return False
+ heapq.heappop(pool.heap)
+ self._evict(pool, victim)
+ return True
+ while not pool.free_slots:
+ victim = next((k for k in pool.cache if k not in pinned), None)
+ if victim is None:
+ self.not_admitted += 1
+ return False
+ self._evict(pool, victim)
+ return True
+
+ def demote(self, prefix):
+ """Move the experts installed for ``prefix`` during a cold call to the cold end of their pool."""
+ cache = self.pool_of[prefix].cache
+ for key in self.cold_installed.pop(prefix, ()):
+ if key in cache:
+ cache.move_to_end(key, last=False)
+
+ def allocate_host_cache(self, host_cache_bytes):
+ """Keep up to ``host_cache_bytes`` of expert bytes in (pinned) host memory, between storage and the device
+ cache: a device-cache miss then costs one host-to-device copy instead of a storage read. Worth it when host
+ memory is larger than device memory (discrete GPUs); on unified memory the device cache already is host RAM."""
+ if self.quantized is None and self.source is None:
+ raise ValueError("host_cache_bytes needs quantized_experts or an expert_source")
+ if len(self.pools) != 1:
+ raise ValueError("host_cache_bytes needs experts of a single layout")
+ self.host_slots = host_cache_bytes // self.slot_bytes
+ if not self.host_slots:
+ return
+ self.host_pool = torch.empty(
+ (self.host_slots, self.slot_bytes), dtype=torch.uint8, pin_memory=self.device.type != "cpu"
+ )
+ self.host_free = list(range(self.host_slots))
+
+ def _host_row(self, tensor):
+ """The host-pool row ``tensor`` is, or None."""
+ if self.host_pool is None:
+ return None
+ offset = tensor.data_ptr() - self.host_pool.data_ptr()
+ if 0 <= offset < self.host_pool.numel() and offset % self.slot_bytes == 0:
+ return offset // self.slot_bytes
+ return None
+
+ def _release_host_row(self, tensor):
+ row = self._host_row(tensor)
+ if row is not None:
+ with self.host_lock:
+ self.host_busy.discard(row)
+ return row is not None
+
+ def _read_bytes(self, prefix, expert, out):
+ """Read one expert's bytes into ``out`` (a CPU uint8 tensor of its pool's slot size)."""
+ if self.source is not None:
+ self.source.read(prefix, expert, out)
+ else:
+ self.quantized[0].read(f"{prefix}.experts", expert, out=out)
+ return out
+
+ def _read_host_cached(self, prefix, expert):
+ """An expert through the host tier: a hit hands out its row, a miss reads into a free (or the least recently
+ used, not busy) row. Returns None when every row is busy. Safe on I/O threads."""
+ key = (prefix, expert)
+ with self.host_lock:
+ row = self.host_cache.get(key)
+ if row is not None:
+ self.host_cache.move_to_end(key)
+ self.host_busy.add(row)
+ self.host_hits += 1
+ return (self.host_pool[row],)
+ if self.host_free:
+ row = self.host_free.pop()
+ else:
+ victim = next((k for k, r in self.host_cache.items() if r not in self.host_busy), None)
+ if victim is None:
+ return None
+ row = self.host_cache.pop(victim)
+ self.host_busy.add(row)
+ self.host_misses += 1
+ try:
+ self._read_bytes(prefix, expert, self.host_pool[row])
+ except BaseException:
+ with self.host_lock:
+ self.host_busy.discard(row)
+ self.host_free.append(row)
+ raise
+ with self.host_lock:
+ self.host_cache[key] = row
+ return (self.host_pool[row],)
+
+ def preload_host(self):
+ """Read every expert into the host tier now (in parallel when there are I/O threads)."""
+ keys = [(prefix, expert) for prefix, table in self.host_slot_tables.items() for expert in range(table.numel())]
+ if self.host_slots < len(keys):
+ raise ValueError(
+ f"host preload needs room for all {len(keys)} experts; the host tier has {self.host_slots}"
+ )
+ if self.executor is not None:
+ results = [f.result() for f in [self.executor.submit(self._read_host_cached, *key) for key in keys]]
+ else:
+ results = [self._read_host_cached(*key) for key in keys]
+ for tensors in results:
+ self._release_host_row(tensors[0])
+ self.host_hits = self.host_misses = 0
+
+ def _read_host(self, prefix, expert):
+ """Read expert bytes or floating weights on the host; safe to run on I/O threads."""
+ if self.host_pool is not None:
+ tensors = self._read_host_cached(prefix, expert)
+ if tensors is not None:
+ return tensors
+ if self.quantized is not None or self.source is not None:
+ # Straight into a recycled pinned buffer: copying pinned memory to the device costs the host ~5x less
+ # than pageable memory (0.03 vs 0.14 ms for a 2 MB expert on an M5).
+ return (self._read_bytes(prefix, expert, self._host_buffer()[: self.pool_of[prefix].slot_bytes]),)
+ return tuple(t.to(self.dtype) for t in self.reader.read_expert(prefix, expert))
+
+ def _host_buffer(self):
+ """A host buffer for one expert (of the largest size, so one size is recycled), pinned when the device can
+ use it; called from I/O threads."""
+ try:
+ return self.free_host_buffers.pop()
+ except IndexError:
+ return torch.empty(self.host_buffer_bytes, dtype=torch.uint8, pin_memory=self.device.type != "cpu")
+
+ def prefetch(self, prefix, experts):
+ """Start reading the experts of ``prefix`` that are neither resident nor already being read."""
+ if self.executor is None:
+ return
+ cache = self.pool_of[prefix].cache
+ for expert in experts:
+ key = (prefix, expert)
+ if key not in cache and key not in self.pending:
+ self.pending[key] = self.executor.submit(self._read_host, prefix, expert)
+ self.prefetched += 1
+
+ def discard(self, prefix):
+ """Drop the reads started for ``prefix`` that its forward call did not use (wrong predictions)."""
+ for key in [key for key in self.pending if key[0] == prefix]:
+ future = self.pending.pop(key)
+ if not future.cancel() and self.host_pool is not None:
+ # Already read (or reading): its host-tier row was handed out and will not be copied.
+ future.add_done_callback(self._release_unused)
+
+ def _release_unused(self, future):
+ if not future.cancelled() and future.exception() is None:
+ self._release_host_row(future.result()[0])
+
+ def _take_host(self, prefix, expert):
+ """One expert in host memory: from a read already started by :meth:`prefetch`, else read now."""
+ future = self.pending.pop((prefix, expert), None)
+ if future is not None and not future.cancelled():
+ self.prefetch_used += 1
+ return future.result()
+ return self._read_host(prefix, expert)
+
+ # TODO(portability): share pinned host staging for asynchronous accelerator copies with _install.
+ def _read(self, prefix, expert):
+ """One expert on the device, for a single use."""
+ tensors = self._take_host(prefix, expert)
+ self.inflight.append(tensors)
+ if self.quantized is not None or self.source is not None:
+ return _split_expert(tensors[0].to(self.device, non_blocking=True), self.pool_of[prefix].layout)
+ return tuple(t.to(self.device, non_blocking=True) for t in tensors)
+
+ def release_host_buffers(self):
+ """Called right after a host synchronization: every queued copy has read its host buffer."""
+ # TODO(portability): validate copy completion and slot reuse on each backend before releasing buffers.
+ if self.quantized is not None or self.source is not None:
+ # list.append / pop are atomic, so I/O threads can take buffers back while this runs.
+ for tensors in self.inflight:
+ if self._release_host_row(tensors[0]):
+ continue # a host-tier row: it stays cached, now free to be evicted
+ buffer = tensors[0] if tensors[0]._base is None else tensors[0]._base
+ if buffer.dtype == torch.uint8 and buffer.numel() == self.host_buffer_bytes:
+ self.free_host_buffers.append(buffer)
+ self.inflight.clear()
+
+ def synchronize(self):
+ """Wait for queued device work, then release the host buffers of its copies."""
+ # TODO(portability): synchronize other accelerator backends.
+ if self.device.type == "mps":
+ torch.mps.synchronize()
+ elif self.device.type == "cuda":
+ torch.cuda.synchronize(self.device)
+ self.release_host_buffers()
+
+ def device_slot_table(self, prefix):
+ """The layer's expert -> slot table on the device, refreshed from the host copy when it changed."""
+ if prefix in self.stale_slot_tables:
+ snapshot = self.host_slot_tables[prefix].clone()
+ self.inflight.append((snapshot,)) # read by a queued non-blocking copy
+ self.slot_tables[prefix].copy_(snapshot, non_blocking=True)
+ self.stale_slot_tables.discard(prefix)
+ return self.slot_tables[prefix]
+
+ def device_slots(self, prefix, routing):
+ """Slots of host-side ``routing`` as a device tensor: one small copy instead of a device lookup."""
+ slots = self.host_slot_tables[prefix][routing]
+ self.inflight.append((slots,))
+ return slots.to(self.device, non_blocking=True)
+
+ def install_ahead(self, prefix, experts):
+ """Install prefetched experts while queued device work runs, without counting lookups."""
+ if not self.materialized:
+ self.materialize()
+ cache = self.pool_of[prefix].cache
+ for expert in experts:
+ key = (prefix, expert)
+ # Only finished reads: waiting here would delay the next layer's launches for guesses that may go unused.
+ if key in cache or key not in self.pending or not self.pending[key].done():
+ continue
+ if self._install(key) is None:
+ break # the pool is full of pinned experts
+ self.installed_ahead += 1
+
+ def _install(self, key, pinned=()):
+ if not self._make_room(key, pinned):
+ return None
+ pool = self.pool_of[key[0]]
+ # Keep host buffers alive until the next host synchronization.
+ tensors = self._take_host(*key)
+ slot = pool.free_slots.pop()
+ self.inflight.append(tensors)
+ if pool.slot_buffer is not None:
+ pool.slot_buffer[slot].copy_(tensors[0], non_blocking=True)
+ elif self.source is not None:
+ offset = 0
+ for bank in pool.banks:
+ size = bank[slot].numel() * bank.element_size()
+ bank[slot].view(-1).view(torch.uint8).copy_(tensors[0][offset : offset + size], non_blocking=True)
+ offset += size
+ else:
+ for bank, tensor in zip(pool.banks, tensors):
+ bank[slot].copy_(tensor, non_blocking=True)
+ self.host_slot_tables[key[0]][key[1]] = slot
+ self.stale_slot_tables.add(key[0])
+ pool.cache[key] = slot
+ if self.cold:
+ self.cold_installed.setdefault(key[0], []).append(key)
+ self.resident_bytes += pool.slot_bytes
+ self.peak_cache_bytes = max(self.peak_cache_bytes, self.resident_bytes)
+ if self.policy == "lfu":
+ self._push(key)
+ return slot
+
+ def acquire(self, prefix, expert, pinned=()):
+ """Make an expert resident in its pool; returns its slot, or None when it runs once without being cached."""
+ if not self.materialized:
+ self.materialize()
+ key = (prefix, expert)
+ self._count(key)
+ self.uses[key] = self.uses.get(key, 0) + 1
+ cache = self.pool_of[prefix].cache
+ if key in cache:
+ self.hits += 1
+ if not self.cold:
+ cache.move_to_end(key)
+ return cache[key]
+ self.misses += 1
+ return self._install(key, pinned)
+
+ def expert_tensors(self, prefix, expert):
+ """The device tensors of one expert without counting a lookup: pool views when resident, else read for this use."""
+ pool = self.pool_of[prefix]
+ slot = pool.cache.get((prefix, expert))
+ if slot is None:
+ return self._read(prefix, expert)
+ return tuple(bank[slot] for bank in pool.banks)
+
+ def get(self, prefix, expert):
+ """Return the device tensors of one expert (pool views when cached, else read for this use only)."""
+ slot = self.acquire(prefix, expert)
+ if slot is None:
+ return self._read(prefix, expert)
+ return tuple(bank[slot] for bank in self.pool_of[prefix].banks)
+
+ def _readers(self):
+ return [self.reader] + ([self.quantized[0]] if self.quantized else []) + ([self.source] if self.source else [])
+
+ def preload(self):
+ """Load every expert now (reads in parallel), when the cache holds them all: no storage reads afterwards."""
+ for pool in self.pools:
+ if pool.num_slots < pool.num_experts:
+ raise ValueError(
+ f"preload needs room for all {pool.num_experts} experts of a pool; it has {pool.num_slots} slots"
+ )
+ for prefix, table in self.host_slot_tables.items():
+ experts = list(range(table.numel()))
+ self.prefetch(prefix, experts)
+ for expert in experts:
+ self.acquire(prefix, expert)
+ self.synchronize()
+ self.hits = self.misses = 0
+
+ def stats(self):
+ return {
+ "hits": self.hits,
+ "misses": self.misses,
+ "evictions": self.evictions,
+ "not_admitted": self.not_admitted,
+ "policy": self.policy,
+ "slots": self.num_slots,
+ "pools": len(self.pools),
+ "prefetched": self.prefetched,
+ "prefetch_used": self.prefetch_used,
+ "installed_ahead": self.installed_ahead,
+ "resident_bytes": self.resident_bytes,
+ "peak_cache_bytes": self.peak_cache_bytes,
+ "read_bytes": sum(reader.read_bytes for reader in self._readers()),
+ "read_calls": sum(reader.read_calls for reader in self._readers()),
+ "host_slots": self.host_slots,
+ "host_hits": self.host_hits,
+ "host_misses": self.host_misses,
+ }
+
+
+class _PoolExperts:
+ """Expose a pool's banks to MoE kernels; slot num_slots uses their expert-parallel sentinel handling."""
+
+ def __init__(self, module, pool):
+ self.module = module
+ self.num_experts = pool.num_slots
+ self.gate_up_proj, self.down_proj = pool.banks
+ self.act_fn = module.act_fn
+ self.has_gate = getattr(module, "has_gate", True)
+ self.has_bias = getattr(module, "has_bias", False)
+ self.is_transposed = getattr(module, "is_transposed", False)
+ self.has_post_expert_norm = getattr(module, "has_post_expert_norm", False)
+ self.post_expert_norm = getattr(module, "post_expert_norm", None)
+ self._is_expert_parallel = True
+
+ def _apply_gate(self, gate_up_out):
+ return self.module._apply_gate(gate_up_out)
+
+
+def _run_expert(module, result, hidden_states, top_k_weights, token, slot, gate_up, down):
+ # Bound temporary activations during prefill.
+ for start in range(0, token.numel(), module.offload_token_chunk_size):
+ indices = token[start : start + module.offload_token_chunk_size]
+ slots = slot[start : start + module.offload_token_chunk_size]
+ activated = module._apply_gate(torch.nn.functional.linear(hidden_states[indices], gate_up))
+ output = torch.nn.functional.linear(activated, down)
+ output = output * top_k_weights[indices, slots, None]
+ result.index_add_(0, indices, output.to(result.dtype))
+
+
+def _split_expert(blob, shapes):
+ """Views of one expert's bytes as its quantized tensors: gate_up (weight, scales, biases), then down."""
+ # TODO(portability): validate uint32 views and strided packed banks on each accelerator backend.
+ tensors, offset = [], 0
+ for shape, dtype in shapes:
+ size = math.prod(shape) * dtype.itemsize
+ if offset % dtype.itemsize:
+ raise ValueError("Misaligned quantized expert layout")
+ tensors.append(blob[..., offset : offset + size].view(dtype).view(*blob.shape[:-1], *shape))
+ offset += size
+ return tuple(tensors)
+
+
+def _load_gather_qmm(device):
+ """MLX's quantized MoE matmul from the Hub (Metal) on MPS; elsewhere the slow PyTorch reference below."""
+ if device.type == "mps":
+ try:
+ from .hub_kernels import get_kernel
+
+ return get_kernel(_MLX_QUANTIZATION_KERNEL[0], version=_MLX_QUANTIZATION_KERNEL[1]).gather_qmm
+ except Exception as error: # the kernels package is optional
+ logger.warning(f"Using the PyTorch reference for quantized experts; the Metal kernel failed: {error}")
+ # TODO(portability): add optimized non-Metal kernels; retain the reference as a correctness fallback.
+ return _gather_qmm_reference
+
+
+def _gather_qmm_reference(x, w, scales, biases, rhs_indices, group_size, bits, **_):
+ """`gather_qmm(x [S, 1, K], w [E, N, K * bits / 32], ..., rhs_indices [S], transpose=True)` -> [S, 1, N]."""
+ from .metal_quantization import _affine_dequantize_tensor
+
+ output = x.new_empty(*x.shape[:-1], w.shape[1])
+ for expert in torch.unique(rhs_indices).tolist():
+ rows = rhs_indices == expert
+ weight = _affine_dequantize_tensor(w[expert], scales[expert], biases[expert], group_size, bits)
+ output[rows] = x[rows] @ weight.to(x.dtype).T
+ return output
+
+
+def _quantized_mlp(module, store, x, banks, rhs, *, sorted_indices=False):
+ """Apply the quantized expert selected by rhs for each input row."""
+ _, bits, group_size = store.quantized
+
+ def matmul(inputs, weight, scales, biases):
+ return store.gather_qmm(
+ inputs[:, None, :],
+ weight,
+ scales,
+ biases,
+ rhs_indices=rhs,
+ transpose=True,
+ group_size=group_size,
+ bits=bits,
+ sorted_indices=sorted_indices,
+ ).squeeze(1)
+
+ return matmul(module._apply_gate(matmul(x, *banks[:3])), *banks[3:])
+
+
+def _sorted_kernel_applies(rows, experts):
+ """Whether MLX selects its batched sorted kernel (at least 16 rows and 4 per expert)."""
+ # TODO(portability): select this threshold and the float32 workaround by kernel/backend.
+ return rows >= 16 and rows >= 4 * experts
+
+
+# Experts per sorted `gather_qmm` call in prefill: bounds the transient bank when a long prompt routes to nearly every
+# expert of a layer (64 x 2.15 MB for Qwen3.8's 3-bit experts instead of 512).
+PREFILL_EXPERT_GROUP = 64
+
+
+def _quantized_sorted_forward(module, store, hidden_states, top_k_index, top_k_weights, experts, banks, result):
+ """Add the routes to ``experts`` (sorted by expert, over ``banks`` holding only them) into ``result`` (float32).
+
+ Uses float32 to avoid incorrect fp16/bf16 results in the v2 M5 kernel."""
+ top_k = top_k_index.shape[1]
+ lookup = torch.full((module.num_experts,), -1, dtype=torch.long, device=hidden_states.device)
+ lookup[torch.tensor(experts, device=hidden_states.device)] = torch.arange(
+ len(experts), device=hidden_states.device
+ )
+ flat = lookup[top_k_index.reshape(-1)]
+ selected = torch.nonzero(flat >= 0).squeeze(1)
+ rhs, permutation = torch.sort(flat[selected], stable=True)
+ order = selected[permutation]
+ banks = [bank.float() if bank.is_floating_point() else bank for bank in banks]
+
+ x = hidden_states.float()[order // top_k]
+ output = _quantized_mlp(module, store, x, banks, rhs, sorted_indices=True)
+ output = output * top_k_weights.reshape(-1)[order, None].float()
+ result.index_add_(0, order // top_k, output)
+
+
+def _quantized_pool_forward(module, store, pool, hidden_states, slots, top_k_weights):
+ """Resident quantized experts through `gather_qmm`; non-resident routes (slot ``num_slots``) contribute zero."""
+ num_tokens, top_k = slots.shape
+ rhs = slots.reshape(-1)
+ valid = rhs < pool.num_slots
+ weights = (top_k_weights.reshape(-1) * valid)[:, None]
+ output = _quantized_mlp(module, store, hidden_states.repeat_interleave(top_k, dim=0), pool.banks, rhs * valid)
+ return (output * weights.to(output.dtype)).view(num_tokens, top_k, -1).sum(dim=1).to(hidden_states.dtype)
+
+
+def _pool_forward(module, store, hidden_states, top_k_index, top_k_weights, slots=None):
+ pool = store.pool_of[module.offload_prefix]
+ if slots is None:
+ slots = store.device_slot_table(module.offload_prefix)[top_k_index]
+ if store.quantized is not None:
+ return _quantized_pool_forward(module, store, pool, hidden_states, slots, top_k_weights)
+ kernel = ExpertsInterface._global_mapping[store.compute]
+ return kernel(_PoolExperts(module, pool), hidden_states, slots, top_k_weights)
+
+
+def _commit_mps(store):
+ """Hand the queued MPS work to the GPU without waiting for it.
+
+ PyTorch's MPS stream keeps small kernels in a command buffer that it commits at the next synchronization, so after
+ each layer's routing copy the GPU would idle while the host encodes the next layer. Recording an event commits the
+ buffer.
+ """
+ # TODO(portability): MPS-only workaround for PyTorch's lazy command-buffer commits; check whether other backends
+ # (CUDA streams submit eagerly) need an equivalent, and drop it once PyTorch's MPS stream commits on its own.
+ if store.device.type == "mps":
+ torch.mps.Event().record()
+
+
+def experts_offload_forward(module, hidden_states, top_k_index, top_k_weights):
+ """The ``experts_offload`` experts implementation: resident experts from the cache, missing ones from disk.
+
+ With an ``expert_compute`` callable, this forward does the routing copy, reads, read-ahead, next-layer guesses and
+ installs, and the callable computes the output: ``expert_compute(module, store, hidden_states, top_k_index,
+ top_k_weights, routing, experts, pinned)``, where ``routing`` is ``top_k_index`` on the host, ``experts`` the sorted
+ routed experts and ``pinned`` the keys made resident for this call (the others are read with
+ ``store.expert_tensors``)."""
+ if module.training or torch.is_grad_enabled():
+ raise RuntimeError("experts_offload is inference-only: use eval() and torch.no_grad() / inference_mode()")
+ store = module.offload_store
+ if hidden_states.device != store.device or hidden_states.dtype != store.dtype:
+ raise ValueError(
+ "Moving/casting a streamed model after loading is unsupported; reload on the target device/dtype"
+ )
+ prefix = module.offload_prefix
+ if store.expert_compute is None and store.all_resident():
+ store.hits += top_k_index.numel()
+ return _pool_forward(module, store, hidden_states, top_k_index, top_k_weights)
+ # Copy routing and next-layer predictions to the host together.
+ next_layer = getattr(module, "offload_next", None)
+ predict = next_layer is not None and store.executor is not None
+ predict = predict and top_k_index.numel() <= module.offload_predict_max_routes
+ after_next = getattr(module, "offload_after_next", None) if predict else None
+ if predict:
+ next_prefix, router = next_layer
+ guesses = [torch.nn.functional.linear(hidden_states, router.weight).topk(router.top_k, dim=-1).indices]
+ if after_next is not None:
+ guesses.append(
+ torch.nn.functional.linear(hidden_states, after_next[1].weight).topk(router.top_k, dim=-1).indices
+ )
+ routes = torch.cat([top_k_index.reshape(-1)] + [guess.reshape(-1) for guess in guesses]).cpu()
+ routing = routes[: top_k_index.numel()].view(top_k_index.shape)
+ else:
+ routing = top_k_index.cpu()
+ # The routing copy also waits for earlier host-to-device copies.
+ store.release_host_buffers()
+ experts = torch.unique(routing).tolist()
+ if experts and not (experts[0] >= 0 and experts[-1] < module.num_experts):
+ raise ValueError("Expert parallelism and sentinel expert IDs are unsupported")
+ store.prefetch(prefix, experts[: store.window])
+ readahead = module.offload_readahead_routes
+ readahead_next = readahead and module.offload_next_prefix is not None and top_k_index.numel() >= readahead
+ if readahead_next:
+ # A long prompt routes to nearly every expert of every layer: read the next layer's while this one computes.
+ store.prefetch(module.offload_next_prefix, range(module.num_experts))
+ width = top_k_index.shape[0] * (next_layer[1].top_k if predict else 0)
+ guessed = torch.unique(routes[top_k_index.numel() : top_k_index.numel() + width]).tolist() if predict else []
+ if predict:
+ store.prefetch(next_prefix, guessed)
+ if after_next is not None:
+ store.prefetch(after_next[0], torch.unique(routes[top_k_index.numel() + width :]).tolist())
+ cold_routes = module.offload_cold_prefill_routes
+ store.cold = bool(cold_routes) and store.policy == "lru" and top_k_index.numel() >= cold_routes
+ try:
+ result = _offloaded_experts(module, store, prefix, hidden_states, top_k_index, top_k_weights, routing, experts)
+ finally:
+ store.discard(prefix)
+ # Also demotes what a previous cold call installed ahead for this layer; `store.cold` stays set until the next
+ # call, so the next layer's installs below are demoted after that layer runs.
+ store.demote(prefix)
+ if store.commit:
+ _commit_mps(store)
+ if guessed:
+ store.install_ahead(next_prefix, guessed)
+ if readahead_next and module.offload_prefill_install_ahead:
+ # The next layer's reads started above: queue the finished ones' copies behind this layer's compute, so that
+ # layer finds them installed instead of copying while the device idles after its routing copy.
+ store.install_ahead(module.offload_next_prefix, range(module.num_experts))
+ return result
+
+
+def _offloaded_experts(module, store, prefix, hidden_states, top_k_index, top_k_weights, routing, experts):
+ pool = store.pool_of[prefix]
+ pinned, uncached = set(), []
+ for position, expert in enumerate(experts):
+ store.prefetch(prefix, experts[position + 1 : position + 1 + store.window])
+ if store.acquire(prefix, expert, pinned) is None:
+ uncached.append(expert)
+ else:
+ pinned.add((prefix, expert))
+ if store.expert_compute is not None:
+ return store.expert_compute(module, store, hidden_states, top_k_index, top_k_weights, routing, experts, pinned)
+ if store.quantized is not None and _sorted_kernel_applies(top_k_index.numel(), len(experts)):
+ # A group of experts at a time: a prefill that routes to every expert never holds a whole layer of them.
+ result = torch.zeros(hidden_states.shape, dtype=torch.float32, device=hidden_states.device)
+ for start in range(0, len(experts), PREFILL_EXPERT_GROUP):
+ group = experts[start : start + PREFILL_EXPERT_GROUP]
+ per_expert = []
+ for position, expert in enumerate(group):
+ if (prefix, expert) in pinned:
+ slot = pool.cache[(prefix, expert)]
+ per_expert.append([bank[slot] for bank in pool.banks])
+ else:
+ store.prefetch(prefix, experts[start + position + 1 : start + position + 1 + store.window])
+ per_expert.append(store._read(prefix, expert))
+ banks = [torch.stack(parts) for parts in zip(*per_expert)]
+ del per_expert
+ _quantized_sorted_forward(module, store, hidden_states, top_k_index, top_k_weights, group, banks, result)
+ if store.inflight:
+ store.synchronize()
+ return result.to(hidden_states.dtype)
+ if pinned:
+ slots = store.device_slots(prefix, routing)
+ result = _pool_forward(module, store, hidden_states, top_k_index, top_k_weights, slots)
+ else:
+ result = torch.zeros_like(hidden_states)
+ for position, expert in enumerate(uncached):
+ store.prefetch(prefix, uncached[position + 1 : position + 1 + store.window])
+ token, slot = (t.to(hidden_states.device) for t in torch.where(routing == expert))
+ tensors = store._read(prefix, expert)
+ if store.quantized is not None:
+ banks = [tensor[None] for tensor in tensors]
+ rhs = torch.zeros_like(token)
+ output = _quantized_mlp(module, store, hidden_states[token], banks, rhs) * top_k_weights[token, slot, None]
+ result.index_add_(0, token, output.to(result.dtype))
+ else:
+ _run_expert(module, result, hidden_states, top_k_weights, token, slot, *tensors)
+ return result
+
+
+def _checkpoint_prefix(reader, name, quantized_reader=None):
+ """Find a unique packed, per-expert, or quantized checkpoint prefix, allowing a different model root."""
+ suffix = name.split(".", 1)[1] if name.startswith("model.") else name
+ found = set()
+ for key in reader.tensors:
+ if key.endswith(f"{suffix}.gate_up_proj"):
+ found.add(key.removesuffix(".gate_up_proj"))
+ elif key.endswith(f"{suffix}.0.gate_proj.weight"):
+ found.add(key.removesuffix(".0.gate_proj.weight"))
+ if not found and quantized_reader is not None:
+ found = {key.removesuffix(".experts") for key in quantized_reader.tensors if key.endswith(f"{suffix}.experts")}
+ if len(found) != 1:
+ raise ValueError(f"Cannot find the checkpoint experts of {name} (candidates: {sorted(found)})")
+ return found.pop()
+
+
+def split_experts_device_map(device_map, experts_offload):
+ """Turn ``device_map`` entries that send the experts to ``"disk"`` into expert streaming.
+
+ An entry whose key names experts modules (``"experts.*"``, ``"model.layers.*.mlp.experts"``, ...) and whose value is
+ ``"disk"`` streams every experts module from disk through the device cache, instead of Accelerate's disk offload
+ (which would read every expert of a layer at each forward). The rest of the map must place the model on one device.
+ Returns the device map without those entries and the ``experts_offload`` options (``cache_bytes`` required)."""
+ if not isinstance(device_map, dict):
+ return device_map, experts_offload
+ keys = [key for key, value in device_map.items() if value == "disk" and "experts" in str(key)]
+ if not keys:
+ return device_map, experts_offload
+ rest = {key: value for key, value in device_map.items() if key not in keys}
+ if not rest:
+ raise ValueError('Map the rest of the model to a device too, e.g. {"": "mps", "experts.*": "disk"}')
+ if "disk" in rest.values():
+ raise ValueError("Streamed experts cannot be combined with disk offload of other modules")
+ options = dict(experts_offload or {})
+ if "cache_bytes" not in options:
+ raise ValueError(
+ 'Experts on "disk" stream through a device cache: pass its size, e.g. experts_offload={"cache_bytes": 8 << 30}'
+ )
+ return rest, options
+
+
+_OPTIONS = {
+ "cache_bytes",
+ "quantized_experts",
+ "cache_policy",
+ "io_threads",
+ "prefetch_depth",
+ "preload",
+ "bypass_page_cache",
+ "experts_implementation",
+ "token_chunk_size",
+ "predict_max_routes",
+ "mps_commit",
+ "host_cache_bytes",
+ "host_preload",
+ "prefill_readahead_routes",
+ "prefill_install_ahead",
+ "cold_prefill_routes",
+ "expert_source",
+ "expert_compute",
+ "preload_profile",
+}
+
+
+def prepare_experts_offload(model, checkpoint_files, *, dtype, device, options):
+ """Replace meta expert banks with a shared disk-backed cache and return checkpoint keys to skip.
+
+ Options:
+ cache_bytes (`int`, required): device memory for cached experts.
+ quantized_experts (`str` or `os.PathLike`, *optional*): a directory written by :func:`quantize_experts`; the
+ experts are then read and cached quantized and run by `gather_qmm`. The checkpoint may omit its own experts.
+ cache_policy (`str`, defaults to `"lru"`): `"lru"` or `"lfu"`.
+ io_threads (`int`, defaults to 4): threads reading experts; 0 reads synchronously and disables prefetch.
+ prefetch_depth (`int`, defaults to 2): how many layers ahead to guess and read experts (0 disables).
+ preload (`bool`, defaults to `False`): read every expert at load time; the cache must hold them all.
+ bypass_page_cache (`bool`, defaults to `False`): keep streamed bytes out of the OS page cache.
+ experts_implementation (`str`, defaults to `"grouped_mm"`): kernel for cached BF16 experts, or `"batched_mm"`.
+ token_chunk_size (`int`, defaults to 256): row chunk for experts that run uncached.
+ predict_max_routes (`int`, defaults to 64): prefetch only for calls with at most this many routes (decode).
+ prefill_readahead_routes (`int`, defaults to 4096): calls with at least this many routes (long prompts) also
+ start reading every expert of the next MoE layer, which such a prompt nearly all uses, so storage keeps
+ reading while the device computes. 0 disables.
+ host_cache_bytes (`int`, defaults to 0): host memory (pinned on accelerators) kept as a tier of quantized
+ experts between storage and the device cache; for discrete GPUs with more host than device memory.
+ host_preload (`bool`, defaults to `False`): read every expert into the host tier at load time.
+ prefill_install_ahead (`bool`, defaults to `False`): with `prefill_readahead_routes`, also install the next
+ layer's finished reads behind this layer's compute.
+ cold_prefill_routes (`int`, defaults to 0): calls with at least this many routes (long prompts) do not refresh
+ the LRU order on hits, and the experts they install move to the cold end once the layer is done, so a long
+ prompt does not evict the experts decoding keeps using. 0 disables. LRU only.
+ mps_commit (`bool`, defaults to `True`): on MPS, commit the queued work after each decoder-layer sub-block and
+ after the experts, so the GPU runs it while the host encodes what follows.
+ expert_source (*optional*): read experts from an external store instead of the checkpoint (see
+ :class:`ExpertStore`); needs `expert_compute`. Layers may differ in layout: each layout gets its own pool and
+ every layer the same number of slots.
+ preload_profile (`dict` or `str`, *optional*): a routing profile (``{prefix: {expert: count}}``, or a JSON file of
+ one, as returned by `model.experts_offload_store.routing_profile()`); the first forward fills each layer's
+ slots with its most used experts, so decoding starts from a warm cache.
+ expert_compute (`Callable`, *optional*): computes the experts output over the pools (see
+ :func:`experts_offload_forward`); required with `expert_source`, where the experts' format is the source's.
+ """
+ options = dict(options)
+ unknown = set(options) - _OPTIONS
+ if unknown or "cache_bytes" not in options:
+ raise ValueError(f"experts_offload needs `cache_bytes`; unknown options: {sorted(unknown)}")
+ if not checkpoint_files or not all(str(f).endswith(".safetensors") for f in checkpoint_files):
+ raise ValueError("experts_offload reads experts from safetensors checkpoints only")
+ if dtype not in _DTYPES.values():
+ raise ValueError("experts_offload needs a floating dtype (float32/float16/bfloat16)")
+ source = options.get("expert_source")
+ if (source is None) != (options.get("expert_compute") is None):
+ raise ValueError("expert_source and expert_compute go together")
+ if options.get("experts_implementation", "grouped_mm") not in ("grouped_mm", "batched_mm"):
+ raise ValueError("experts_offload supports experts_implementation 'grouped_mm' or 'batched_mm'")
+ ExpertsInterface.register("experts_offload", experts_offload_forward)
+ device = torch.empty(0, device=device).device
+ reader = SafetensorsReader(Path(checkpoint_files[0]).parent, options.get("bypass_page_cache", False))
+ store = ExpertStore(
+ reader,
+ options["cache_bytes"],
+ device,
+ dtype,
+ policy=options.get("cache_policy", "lru"),
+ io_threads=options.get("io_threads", 4),
+ quantized=options.get("quantized_experts"),
+ source=source,
+ )
+ store.compute = options.get("experts_implementation", "grouped_mm")
+ store.expert_compute = options.get("expert_compute")
+ store.preload_profile = options.get("preload_profile")
+ experts = [
+ (name, module)
+ for name, module in model.named_modules()
+ if hasattr(module, "_apply_gate") and isinstance(getattr(module, "gate_up_proj", None), torch.nn.Parameter)
+ ]
+ if not experts:
+ raise ValueError(f"{type(model).__name__} has no experts module that can be offloaded")
+ consumed = set()
+ for name, module in experts:
+ if module.has_bias or module.is_transposed or not module.is_concatenated or module.has_post_expert_norm:
+ raise ValueError(
+ f"experts_offload supports gated, concatenated, untransposed experts without bias: {name}"
+ )
+ if source is not None:
+ prefix = source.prefix(name) if hasattr(source, "prefix") else name
+ else:
+ prefix = _checkpoint_prefix(reader, name, store.quantized[0] if store.quantized else None)
+ consumed.update(store.register(prefix, module))
+ module.register_parameter("gate_up_proj", None)
+ module.register_parameter("down_proj", None)
+ module._is_hf_initialized = True # Banks remain in the checkpoint.
+ module.config._experts_implementation = "experts_offload"
+ module.offload_store = store
+ module.offload_prefix = prefix
+ module.offload_token_chunk_size = options.get("token_chunk_size", 256)
+ module.offload_predict_max_routes = options.get("predict_max_routes", 64)
+ module.offload_next = module.offload_after_next = None
+ module.offload_readahead_routes = options.get("prefill_readahead_routes", 4096)
+ module.offload_prefill_install_ahead = options.get("prefill_install_ahead", False)
+ module.offload_cold_prefill_routes = options.get("cold_prefill_routes", 0)
+ store.allocate(materialize=False)
+ depth = options.get("prefetch_depth", 2)
+ routers = []
+ for name, module in experts:
+ router = getattr(model.get_submodule(name.rpartition(".")[0]), "gate", None)
+ usable = router is not None and hasattr(router, "weight") and hasattr(router, "top_k")
+ routers.append((module.offload_prefix, router) if usable else None)
+ for index, (_, module) in enumerate(experts):
+ module.offload_next_prefix = experts[index + 1][1].offload_prefix if index + 1 < len(experts) else None
+ if depth >= 1 and index + 1 < len(routers):
+ module.offload_next = routers[index + 1]
+ if depth >= 2 and index + 2 < len(routers):
+ module.offload_after_next = routers[index + 2]
+ if options.get("mps_commit", True):
+ store.commit = True
+
+ def commit_hook(module, args, output):
+ _commit_mps(store) # returns nothing: a forward hook's return value would replace the output
+
+ for name, _ in experts:
+ layer = model.get_submodule(name.rpartition(".")[0].rpartition(".")[0])
+ for child in layer.children():
+ child.register_forward_hook(commit_hook)
+ if options.get("host_cache_bytes", 0):
+ store.allocate_host_cache(options["host_cache_bytes"])
+ if options.get("host_preload", False):
+ store.preload_host()
+ if options.get("preload", False):
+ store.preload()
+ model.experts_offload_store = store
+ return consumed
+
+
+def quantize_experts(checkpoint, output, *, bits=4, group_size=64, device="cpu"):
+ """Write MLX affine experts as uint8 rows: gate_up then down, each with weights, scales and biases.
+
+ Uses the Metal kernel on MPS, otherwise PyTorch. Peak memory is one experts module."""
+ from safetensors.torch import save_file
+
+ from .metal_quantization import _affine_quantize_tensor
+
+ device = torch.device(device)
+ quantize = lambda weight: _affine_quantize_tensor(weight, group_size, bits) # noqa: E731
+ quantizer = "pytorch-reference"
+ # TODO(portability): Metal quantizer on MPS only; other accelerators use the PyTorch reference.
+ if device.type == "mps":
+ from .hub_kernels import get_kernel
+
+ kernel = get_kernel(_MLX_QUANTIZATION_KERNEL[0], version=_MLX_QUANTIZATION_KERNEL[1])
+ quantize = lambda weight: kernel.quantize(weight, group_size=group_size, bits=bits) # noqa: E731
+ quantizer = "mlx-quantization-metal-kernels"
+ reader = SafetensorsReader(checkpoint)
+ output = Path(output)
+ output.mkdir(parents=True, exist_ok=True)
+ packed = sorted(name.removesuffix(".gate_up_proj") for name in reader.tensors if name.endswith(".gate_up_proj"))
+ per_expert = sorted(
+ name.removesuffix(".0.gate_proj.weight") for name in reader.tensors if name.endswith(".0.gate_proj.weight")
+ )
+ weight_map = {}
+ for index, prefix in enumerate(packed + per_expert):
+ if prefix in packed:
+ experts = reader.shape(f"{prefix}.gate_up_proj")[0]
+ else:
+ experts = sum(
+ 1 for name in reader.tensors if re.fullmatch(rf"{re.escape(prefix)}\.\d+\.gate_proj\.weight", name)
+ )
+
+ blobs = []
+ for expert in range(experts):
+ parts = []
+ for weight in reader.read_expert(prefix, expert):
+ weight_q, scales, biases = quantize(weight.to(device))
+ parts += [weight_q, scales.to(weight.dtype), biases.to(weight.dtype)]
+ blobs.append(torch.cat([part.cpu().contiguous().view(torch.uint8).reshape(-1) for part in parts]))
+ filename = f"experts-{index:05d}.safetensors"
+ save_file({f"{prefix}.experts": torch.stack(blobs)}, output / filename, {"format": "pt"})
+ weight_map[f"{prefix}.experts"] = filename
+ logger.info(f"quantized {prefix}: {experts} experts")
+ (output / "model.safetensors.index.json").write_text(json.dumps({"metadata": {}, "weight_map": weight_map}))
+ info = {"format": _QUANTIZED_FORMAT, "bits": bits, "group_size": group_size, "quantizer": quantizer}
+ (output / "quantization.json").write_text(json.dumps(info, indent=2))
+ return output
diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index 99aef4a573..0cbc5cef9e 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -74,6 +74,7 @@ from .integrations.accelerate import (
)
from .integrations.deepspeed import _load_state_dict_into_zero3_model, prevent_arange_dtype_change
from .integrations.eager_paged import eager_paged_attention_forward
+from .integrations.experts_offload import prepare_experts_offload, split_experts_device_map
from .integrations.finegrained_fp8 import ALL_FP8_EXPERTS_FUNCTIONS
from .integrations.flash_attention import flash_attention_forward
from .integrations.flex_attention import flex_attention_forward
@@ -187,6 +188,8 @@ class LoadStateDictConfig:
weights_only: bool = True
weight_mapping: list[WeightConverter | WeightRenaming] | None = None
disable_mmap: bool | None = None
+ # Checkpoint keys loaded by another component.
+ skip_keys: set[str] = field(default_factory=set)
@property
def is_quantized(self) -> bool:
@@ -4099,6 +4102,11 @@ class PreTrainedModel(
This should only be used as an inference optimization, as it can slightly change outputs. If omitted,
`from_pretrained()` falls back to `config.fusion_config` when available. Refer to the fusion mapping
guide in `docs/source/en/fusion_mapping.md` for more details.
+ experts_offload (`dict`, *optional*):
+ Options for MoE experts streamed from disk through a device cache, for inference: either with a
+ `device_map` entry mapping the experts to `"disk"` (`{"": "mps", "experts.*": "disk"}`), or alone on a
+ single-device map. Requires `cache_bytes`; see
+ `transformers.integrations.experts_offload.prepare_experts_offload` for the options.
key_mapping (`dict[str, str], *optional*):
A potential mapping of the weight names if using a model on the Hub which is compatible to a Transformers
architecture, but was not converted accordingly.
@@ -4166,6 +4174,7 @@ class PreTrainedModel(
use_kernels = kwargs.pop("use_kernels", False)
kernel_config = kwargs.pop("kernel_config", None)
key_mapping = kwargs.pop("key_mapping", None)
+ experts_offload = kwargs.pop("experts_offload", None)
# Not used anymore -- remove them from the kwargs
for name in ["mirror", "_fast_init", "low_cpu_mem_usage", "from_tf", "from_flax", "offload_state_dict"]:
@@ -4262,6 +4271,8 @@ class PreTrainedModel(
cache_dir=cache_dir,
)
download_kwargs["revision"] = revision
+ # `{"experts.*": "disk"}` entries stream the experts (see `split_experts_device_map`) instead of disk offload.
+ device_map, experts_offload = split_experts_device_map(device_map, experts_offload)
device_map = check_and_set_device_map(device_map) # warn, error and fix the device map
user_agent = {"file_type": "model", "framework": "pytorch", "from_auto_class": from_auto_class}
@@ -4394,6 +4405,20 @@ class PreTrainedModel(
if gguf_file:
state_dict = hf_quantizer.get_state_dict(checkpoint_files[0], model)
+ skip_keys = set()
+ if experts_offload is not None:
+ devices = set(device_map.values()) if isinstance(device_map, dict) else {device_map or "cpu"}
+ # Metal quantization only converts nn.Linear modules, never the expert banks this removes.
+ dense_quantizer = hf_quantizer is not None and hf_quantizer.quantization_config.quant_method == "metal"
+ if len(devices) != 1 or gguf_file or (hf_quantizer is not None and not dense_quantizer) or state_dict:
+ raise ValueError(
+ "experts_offload needs a single-device device_map and a safetensors checkpoint, unquantized or "
+ "with MetalConfig for its dense layers"
+ )
+ skip_keys = prepare_experts_offload(
+ model, checkpoint_files, dtype=dtype, device=devices.pop(), options=experts_offload
+ )
+
# Create the dtype_plan to potentially use the `keep_in_fp32` flags (this needs to be called on the already
# instantiated model, as the flags can be modified by instances sometimes)
dtype_plan = model._get_dtype_plan(dtype)
@@ -4425,6 +4450,7 @@ class PreTrainedModel(
use_safetensors=use_safetensors,
download_kwargs=download_kwargs,
disable_mmap=disable_mmap,
+ skip_keys=skip_keys,
)
if distributed_checkpoint_dir is not None:
loading_info = model._load_and_finalize_distributed_pretrained_model(
@@ -4551,6 +4577,8 @@ class PreTrainedModel(
merged_state_dict = {}
for file in checkpoint_files:
if load_config.disable_mmap or _is_on_hf_mount(file):
+ if load_config.skip_keys:
+ raise ValueError("experts_offload does not support disable_mmap or HF mounts")
with open(file, "rb") as _fh:
merged_state_dict.update(_safe_load_bytes(_fh.read()))
continue
@@ -4570,7 +4598,8 @@ class PreTrainedModel(
file_pointer = safe_open(file, framework="pt", device=device, backend=backend)
all_pointer.add(file_pointer)
for k in file_pointer.keys():
- merged_state_dict[k] = file_pointer.get_slice(k) # don't materialize yet
+ if k not in load_config.skip_keys:
+ merged_state_dict[k] = file_pointer.get_slice(k) # don't materialize yet
# Checkpoints are .bin
elif checkpoint_files is not None:
merged_state_dict = {}
diff --git a/tests/integrations/test_experts_offload.py b/tests/integrations/test_experts_offload.py
new file mode 100644
index 0000000000..ec02004e43
--- /dev/null
+++ b/tests/integrations/test_experts_offload.py
@@ -0,0 +1,535 @@
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import math
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+from transformers import is_torch_available
+from transformers.testing_utils import require_torch
+
+
+if is_torch_available():
+ import torch
+ from safetensors.torch import load_file, save_file
+
+ from transformers import OlmoeConfig, OlmoeForCausalLM, Qwen3MoeConfig, Qwen3MoeForCausalLM
+ from transformers.integrations.experts_offload import (
+ ExpertStore,
+ SafetensorsReader,
+ _split_expert,
+ quantize_experts,
+ )
+ from transformers.integrations.metal_quantization import _affine_dequantize_tensor
+
+
+HIDDEN, INTERMEDIATE, EXPERTS = 16, 8, 4
+EXPERT_BYTES = 3 * HIDDEN * INTERMEDIATE * 4 # float32
+IDS = [[1, 7, 3, 9, 2, 11, 5]]
+
+
+def tiny_qwen3_moe(seed=0):
+ torch.manual_seed(seed)
+ config = Qwen3MoeConfig(
+ vocab_size=64,
+ hidden_size=HIDDEN,
+ intermediate_size=24,
+ moe_intermediate_size=INTERMEDIATE,
+ num_hidden_layers=2,
+ num_attention_heads=2,
+ num_key_value_heads=1,
+ head_dim=8,
+ num_experts=EXPERTS,
+ num_experts_per_tok=2,
+ )
+ config._attn_implementation = "eager"
+ return Qwen3MoeForCausalLM(config).eval()
+
+
+def greedy(model, ids, tokens=6):
+ with torch.no_grad():
+ return model.generate(torch.tensor(ids), max_new_tokens=tokens, do_sample=False, pad_token_id=0)
+
+
+class PaddedSource:
+ """An external expert store over a model's experts: float32 gate_up and down, plus (layer 1) a padding tensor,
+ so the two layers differ in size and get one pool each."""
+
+ def __init__(self, model):
+ self.experts = {}
+ for index, layer in enumerate(model.model.layers):
+ experts = layer.mlp.experts
+ self.experts[f"source.{index}"] = (experts.gate_up_proj.detach(), experts.down_proj.detach(), index)
+ self.prefixes = list(self.experts)
+ self.read_bytes = self.read_calls = 0
+
+ def prefix(self, module_name):
+ return f"source.{module_name.split('.')[2]}"
+
+ def layout(self, prefix):
+ gate_up, down, index = self.experts[prefix]
+ layout = [(tuple(gate_up.shape[1:]), torch.float32), (tuple(down.shape[1:]), torch.float32)]
+ return layout + ([((16,), torch.uint8)] if index else [])
+
+ def expert_bytes(self, prefix):
+ return sum(math.prod(shape) * dtype.itemsize for shape, dtype in self.layout(prefix))
+
+ def read(self, prefix, expert, out):
+ gate_up, down, _ = self.experts[prefix]
+ blob = torch.cat([gate_up[expert].reshape(-1).view(torch.uint8), down[expert].reshape(-1).view(torch.uint8)])
+ out[: blob.numel()] = blob
+ out[blob.numel() :] = 7
+ self.read_bytes += out.numel()
+ self.read_calls += 1
+
+
+def reference_compute(module, store, hidden_states, top_k_index, top_k_weights, routing, experts, pinned):
+ """`expert_compute` for :class:`PaddedSource`: every route through `store.expert_tensors`."""
+ result = torch.zeros_like(hidden_states)
+ prefix = module.offload_prefix
+ for expert in experts:
+ gate_up, down, *padding = store.expert_tensors(prefix, expert)
+ if padding:
+ assert bool((padding[0] == 7).all())
+ token, slot = (t.to(hidden_states.device) for t in torch.where(routing == expert))
+ activated = module._apply_gate(torch.nn.functional.linear(hidden_states[token], gate_up))
+ output = torch.nn.functional.linear(activated, down) * top_k_weights[token, slot, None]
+ result.index_add_(0, token, output.to(result.dtype))
+ return result
+
+
+# TODO(portability): run on Linux/macOS/Windows and parameterize available devices; current loads use CPU.
+@require_torch
+class ExpertsOffloadTest(unittest.TestCase):
+ def load(self, path, **options):
+ return Qwen3MoeForCausalLM.from_pretrained(
+ path, attn_implementation="eager", experts_offload={"cache_bytes": 3 * EXPERT_BYTES, **options}
+ )
+
+ def test_loads_without_expert_banks_and_matches_eager(self):
+ reference = tiny_qwen3_moe()
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ model, info = Qwen3MoeForCausalLM.from_pretrained(
+ directory,
+ attn_implementation="eager",
+ experts_offload={"cache_bytes": 3 * EXPERT_BYTES},
+ output_loading_info=True,
+ )
+ self.assertEqual((info["missing_keys"], info["unexpected_keys"]), (set(), set()))
+ experts = model.model.layers[0].mlp.experts
+ self.assertIsNone(experts.gate_up_proj)
+ self.assertEqual(experts.config._experts_implementation, "experts_offload")
+ self.assertFalse(any(name.endswith(("gate_up_proj", "down_proj")) for name in model.state_dict()))
+ self.assertIsNone(model.experts_offload_store.pools[0].banks) # device pools are taken at the first use
+ torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+ self.assertGreater(model.experts_offload_store.misses, 0)
+
+ def test_every_cache_size_and_kernel_matches_eager(self):
+ reference = tiny_qwen3_moe(1)
+ expected = greedy(reference, IDS)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ # Cover uncached, partial-cache and fully resident execution.
+ for slots in (0, 1, 3, 8):
+ for implementation in ("grouped_mm", "batched_mm"):
+ with self.subTest(slots=slots, implementation=implementation):
+ model = self.load(
+ directory, cache_bytes=slots * EXPERT_BYTES, experts_implementation=implementation
+ )
+ store = model.experts_offload_store
+ self.assertEqual(store.num_slots, slots)
+ torch.testing.assert_close(greedy(model, IDS), expected)
+ self.assertLessEqual(store.resident_bytes, slots * EXPERT_BYTES)
+ self.assertEqual(store.all_resident(), slots == 8)
+
+ def test_reads_hits_evictions_and_zero_budget(self):
+ reference = tiny_qwen3_moe(2)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ for budget in (0, EXPERT_BYTES):
+ with self.subTest(budget=budget):
+ # No I/O threads: no reads ahead, so the counts below are exact.
+ model = self.load(directory, cache_bytes=budget, io_threads=0)
+ store = model.experts_offload_store
+ module, eager = model.model.layers[0].mlp.experts, reference.model.layers[0].mlp.experts
+ hidden, weights = torch.randn(2, HIDDEN), torch.ones(2, 1)
+ ids = torch.zeros(2, 1, dtype=torch.long)
+ with torch.inference_mode():
+ torch.testing.assert_close(module(hidden, ids, weights), eager(hidden, ids, weights))
+ self.assertEqual(store.stats()["read_bytes"], EXPERT_BYTES)
+ module(hidden, ids, weights)
+ self.assertEqual(store.hits, int(budget > 0))
+ module(hidden, ids + 1, weights)
+ torch.testing.assert_close(module(hidden, ids, weights), eager(hidden, ids, weights))
+ self.assertEqual(store.misses, 3 if budget else 4)
+ self.assertLessEqual(store.resident_bytes, budget)
+
+ def test_prefetch_threads_and_depths_match_eager(self):
+ reference = tiny_qwen3_moe(3)
+ expected = greedy(reference, IDS, tokens=8)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ for io_threads, depth in ((0, 0), (1, 1), (4, 1), (4, 2)):
+ with self.subTest(io_threads=io_threads, depth=depth):
+ model = self.load(directory, io_threads=io_threads, prefetch_depth=depth)
+ layers = model.model.layers
+ if depth:
+ self.assertIs(layers[0].mlp.experts.offload_next[1], layers[1].mlp.gate)
+ self.assertIsNone(layers[1].mlp.experts.offload_next)
+ torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+ store = model.experts_offload_store
+ self.assertEqual(store.pending, {})
+ if io_threads:
+ self.assertGreater(store.prefetch_used, 0)
+ else:
+ self.assertEqual(store.prefetched, 0)
+
+ def test_prefill_readahead_matches_eager(self):
+ reference = tiny_qwen3_moe(3)
+ expected = greedy(reference, IDS, tokens=8)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ # The 7-token prompt has 14 routes per layer, decode steps 2: a threshold of 8 reads ahead in prefill only.
+ for routes in (0, 8):
+ with self.subTest(prefill_readahead_routes=routes):
+ model = self.load(directory, io_threads=2, prefetch_depth=0, prefill_readahead_routes=routes)
+ layers = model.model.layers
+ next_prefix = layers[1].mlp.experts.offload_prefix
+ self.assertEqual(layers[0].mlp.experts.offload_next_prefix, next_prefix)
+ self.assertIsNone(layers[1].mlp.experts.offload_next_prefix)
+ store = model.experts_offload_store
+ calls = []
+ prefetch = store.prefetch
+
+ def spy(prefix, experts):
+ calls.append((prefix, experts))
+ return prefetch(prefix, experts)
+
+ with mock.patch.object(store, "prefetch", side_effect=spy):
+ torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+ self.assertEqual(store.pending, {})
+ readahead = [call for call in calls if call == (next_prefix, range(EXPERTS))]
+ # Once per forward of layer 0 with enough routes: the prompt, not the decode steps.
+ self.assertEqual(len(readahead), 1 if routes else 0)
+
+ def test_mps_commit_hooks(self):
+ reference = tiny_qwen3_moe(3)
+ expected = greedy(reference, IDS, tokens=8)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ for enabled in (True, False):
+ with self.subTest(mps_commit=enabled):
+ with mock.patch("transformers.integrations.experts_offload._commit_mps") as commit:
+ model = self.load(directory, mps_commit=enabled)
+ torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+ self.assertEqual(commit.called, enabled)
+
+ def test_preload_and_fully_resident_layers_skip_the_host_copy(self):
+ reference = tiny_qwen3_moe(4)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ with self.assertRaises(ValueError):
+ self.load(directory, preload=True) # 3 slots cannot hold 8 experts
+ model = self.load(directory, cache_bytes=8 * EXPERT_BYTES, preload=True)
+ store = model.experts_offload_store
+ self.assertTrue(store.all_resident())
+ read_bytes = store.stats()["read_bytes"]
+ original = torch.Tensor.cpu
+ with mock.patch.object(torch.Tensor, "cpu", autospec=True, side_effect=original) as cpu:
+ torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+ self.assertFalse(any(call.args[0].dtype == torch.long for call in cpu.call_args_list))
+ self.assertEqual((store.misses, store.stats()["read_bytes"]), (0, read_bytes))
+
+ def test_per_expert_checkpoint_layout(self):
+ torch.manual_seed(5)
+ config = OlmoeConfig(
+ vocab_size=64,
+ hidden_size=HIDDEN,
+ intermediate_size=INTERMEDIATE,
+ num_hidden_layers=2,
+ num_attention_heads=2,
+ num_key_value_heads=2,
+ num_experts=EXPERTS,
+ num_experts_per_tok=2,
+ )
+ config._attn_implementation = "eager"
+ reference = OlmoeForCausalLM(config).eval()
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory)
+ reference.save_pretrained(path)
+ # Match the Hub checkpoint: one tensor per expert and projection.
+ tensors = load_file(path / "model.safetensors")
+ for name in [name for name in tensors if name.endswith("experts.gate_up_proj")]:
+ prefix = name.removesuffix(".gate_up_proj")
+ gate_up, down = tensors.pop(name), tensors.pop(f"{prefix}.down_proj")
+ for expert in range(EXPERTS):
+ gate, up = gate_up[expert].chunk(2)
+ tensors[f"{prefix}.{expert}.gate_proj.weight"] = gate.contiguous()
+ tensors[f"{prefix}.{expert}.up_proj.weight"] = up.contiguous()
+ tensors[f"{prefix}.{expert}.down_proj.weight"] = down[expert].contiguous()
+ save_file(tensors, path / "model.safetensors", {"format": "pt"})
+ model = OlmoeForCausalLM.from_pretrained(
+ path, attn_implementation="eager", experts_offload={"cache_bytes": 3 * EXPERT_BYTES}
+ )
+ torch.testing.assert_close(greedy(model, IDS), greedy(reference, IDS))
+
+ def test_quantized_experts_match_the_dequantized_model(self):
+ reference = tiny_qwen3_moe(6)
+ # 12 tokens x top-2 over 4 experts: prefill takes the sorted path, decode the pool path.
+ ids = [[1, 7, 3, 9, 2, 11, 5, 13, 4, 8, 6, 10]]
+ group = 8
+ shapes = [
+ ((2 * INTERMEDIATE, HIDDEN * 8 // 32), torch.uint32),
+ ((2 * INTERMEDIATE, HIDDEN // group), torch.float32),
+ ((2 * INTERMEDIATE, HIDDEN // group), torch.float32),
+ ((HIDDEN, INTERMEDIATE * 8 // 32), torch.uint32),
+ ((HIDDEN, INTERMEDIATE // group), torch.float32),
+ ((HIDDEN, INTERMEDIATE // group), torch.float32),
+ ]
+ with tempfile.TemporaryDirectory() as directory:
+ path, quantized = Path(directory) / "model", Path(directory) / "experts-q8"
+ reference.save_pretrained(path)
+ quantize_experts(path, quantized, bits=8, group_size=group)
+ # The reference runs the dequantized experts.
+ with torch.no_grad():
+ for index, layer in enumerate(reference.model.layers):
+ blobs = load_file(quantized / f"experts-{index:05d}.safetensors")
+ blobs = blobs[f"model.layers.{index}.mlp.experts.experts"]
+ parts = [_split_expert(blobs[expert], shapes) for expert in range(EXPERTS)]
+ for offset, projection in ((0, "gate_up_proj"), (3, "down_proj")):
+ dequantized = [_affine_dequantize_tensor(*p[offset : offset + 3], group, 8) for p in parts]
+ getattr(layer.mlp.experts, projection).copy_(torch.stack(dequantized))
+ expected = greedy(reference, ids)
+ for cache_bytes in (1000, 3000, 8000):
+ with self.subTest(cache_bytes=cache_bytes):
+ model = self.load(path, cache_bytes=cache_bytes, quantized_experts=quantized)
+ torch.testing.assert_close(greedy(model, ids), expected)
+ # The quantized experts can be the only copy: drop the BF16 experts from the checkpoint.
+ tensors = {k: v for k, v in load_file(path / "model.safetensors").items() if ".mlp.experts." not in k}
+ save_file(tensors, path / "model.safetensors", {"format": "pt"})
+ model = self.load(path, cache_bytes=3000, quantized_experts=quantized)
+ torch.testing.assert_close(greedy(model, ids), expected)
+
+ def test_host_tier(self):
+ reference = tiny_qwen3_moe(6)
+ ids = [[1, 7, 3, 9, 2, 11, 5, 13, 4, 8, 6, 10]]
+ with tempfile.TemporaryDirectory() as directory:
+ path, quantized = Path(directory) / "model", Path(directory) / "experts-q8"
+ reference.save_pretrained(path)
+ quantize_experts(path, quantized, bits=8, group_size=8)
+ expected = greedy(self.load(path, cache_bytes=3000, quantized_experts=quantized), ids)
+ slot_bytes = self.load(
+ path, cache_bytes=3000, quantized_experts=quantized
+ ).experts_offload_store.slot_bytes
+ total = 2 * EXPERTS # two layers
+ for host_experts, preload, io_threads in (
+ (total, False, 2),
+ (3, False, 2),
+ (total, True, 0),
+ (total, True, 4),
+ ):
+ with self.subTest(host_experts=host_experts, preload=preload, io_threads=io_threads):
+ model = self.load(
+ path, cache_bytes=3000, quantized_experts=quantized, io_threads=io_threads,
+ host_cache_bytes=host_experts * slot_bytes, host_preload=preload,
+ ) # fmt: skip
+ store = model.experts_offload_store
+ self.assertEqual(store.host_slots, host_experts)
+ reads = store.stats()["read_calls"]
+ self.assertEqual(reads, total if preload else 0)
+ torch.testing.assert_close(greedy(model, ids), expected)
+ torch.testing.assert_close(greedy(model, ids), expected)
+ if host_experts == total:
+ # Every expert was read at most once: device-cache misses are served from host memory.
+ self.assertEqual(store.stats()["read_calls"], total)
+ self.assertGreater(store.host_hits, 0)
+ store.synchronize() # rows of the last copies are released at the next synchronization
+ self.assertEqual(store.host_busy, set()) # every handed-out row was released
+ with self.assertRaises(ValueError):
+ self.load(path, cache_bytes=3000, host_cache_bytes=1 << 20) # needs quantized experts
+
+ def test_device_map_sends_experts_to_disk(self):
+ reference = tiny_qwen3_moe(10)
+ expected = greedy(reference, IDS)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ for key in ("experts.*", "model.layers.*.mlp.experts"):
+ with self.subTest(key=key):
+ model = Qwen3MoeForCausalLM.from_pretrained(
+ directory,
+ attn_implementation="eager",
+ device_map={"": "cpu", key: "disk"},
+ experts_offload={"cache_bytes": 3 * EXPERT_BYTES},
+ )
+ experts = model.model.layers[0].mlp.experts
+ self.assertIsNone(experts.gate_up_proj)
+ self.assertEqual(experts.config._experts_implementation, "experts_offload")
+ torch.testing.assert_close(greedy(model, IDS), expected)
+ self.assertGreater(model.experts_offload_store.misses, 0)
+ for device_map, options in (
+ ({"experts.*": "disk"}, {"cache_bytes": EXPERT_BYTES}), # nothing on a device
+ ({"": "cpu", "experts.*": "disk"}, None), # no cache size
+ ):
+ with self.subTest(device_map=device_map), self.assertRaises(ValueError):
+ Qwen3MoeForCausalLM.from_pretrained(directory, device_map=device_map, experts_offload=options)
+
+ def test_routing_profile_preload(self):
+ reference = tiny_qwen3_moe(11)
+ expected = greedy(reference, IDS)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ model = self.load(directory, cache_bytes=4 * EXPERT_BYTES, io_threads=0)
+ greedy(model, IDS)
+ profile = model.experts_offload_store.routing_profile()
+ self.assertEqual(
+ sum(map(sum, (p.values() for p in profile.values()))), model.experts_offload_store.lookups
+ )
+ path = Path(directory) / "profile.json"
+ path.write_text(json.dumps(profile))
+ for preload in (profile, str(path)):
+ model = self.load(directory, cache_bytes=4 * EXPERT_BYTES, io_threads=2, preload_profile=preload)
+ store = model.experts_offload_store
+ self.assertIsNone(store.pools[0].banks) # nothing read before the first forward
+ torch.testing.assert_close(greedy(model, IDS), expected)
+ for prefix, counts in profile.items():
+ top = sorted(counts, key=counts.get, reverse=True)[:2] # 4 slots, 2 layers
+ self.assertTrue(all((prefix, expert) in store.cache for expert in top[:1]))
+ self.assertLess(store.misses, model.experts_offload_store.lookups)
+
+ def test_expert_source_with_mixed_layouts(self):
+ reference = tiny_qwen3_moe(8)
+ expected = greedy(reference, IDS, tokens=8)
+ source = PaddedSource(reference)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ sizes = [source.expert_bytes(prefix) for prefix in source.prefixes]
+ self.assertNotEqual(*sizes) # one pool per layer
+ for io_threads, depth, slots in ((0, 0, 0), (0, 0, 2), (2, 1, 2), (2, 2, EXPERTS)):
+ with self.subTest(io_threads=io_threads, depth=depth, slots=slots):
+ model = self.load(
+ directory, cache_bytes=slots * sum(sizes), io_threads=io_threads, prefetch_depth=depth,
+ expert_source=source, expert_compute=reference_compute,
+ ) # fmt: skip
+ store = model.experts_offload_store
+ self.assertEqual(len(store.pools), 2)
+ self.assertEqual([pool.num_slots for pool in store.pools], [slots, slots])
+ self.assertEqual(model.model.layers[1].mlp.experts.offload_prefix, source.prefixes[1])
+ torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+ self.assertEqual(store.pending, {})
+ self.assertGreater(store.stats()["read_calls"], 0)
+ for pool in store.pools:
+ self.assertLessEqual(len(pool.cache), pool.num_slots)
+ if slots == EXPERTS:
+ self.assertEqual(store.misses, 2 * EXPERTS - store.installed_ahead)
+ with self.assertRaises(ValueError):
+ self.load(directory, expert_source=source) # needs expert_compute
+
+ def test_cold_prefill_routes_matches_eager(self):
+ reference = tiny_qwen3_moe(9)
+ expected = greedy(reference, IDS, tokens=8)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ # 14 routes per layer in the 7-token prompt, 2 per decode step.
+ model = self.load(
+ directory, io_threads=2, prefetch_depth=0, cold_prefill_routes=8, prefill_readahead_routes=8,
+ prefill_install_ahead=True,
+ ) # fmt: skip
+ torch.testing.assert_close(greedy(model, IDS, tokens=8), expected)
+ store = model.experts_offload_store
+ self.assertEqual(store.cold_installed, {})
+ self.assertFalse(store.cold)
+
+ def test_inference_only_and_option_validation(self):
+ reference = tiny_qwen3_moe(7)
+ with tempfile.TemporaryDirectory() as directory:
+ reference.save_pretrained(directory)
+ with self.assertRaises(ValueError):
+ self.load(directory, cache_size=1)
+ with self.assertRaises(ValueError):
+ Qwen3MoeForCausalLM.from_pretrained(directory, experts_offload={"io_threads": 2})
+ with self.assertRaises(ValueError):
+ self.load(directory, experts_implementation="eager")
+ model = self.load(directory)
+ with self.assertRaises(RuntimeError):
+ model(torch.tensor(IDS)) # grad enabled
+ model.train()
+ with torch.no_grad(), self.assertRaises(RuntimeError):
+ model(torch.tensor(IDS))
+
+
+@require_torch
+class ExpertStorePolicyTest(unittest.TestCase):
+ def make_store(self, policy, cache_experts, experts=5, hidden=4, intermediate=2):
+ banks = {
+ "layer.gate_up_proj": torch.arange(experts * 2 * intermediate * hidden, dtype=torch.float32).view(
+ experts, 2 * intermediate, hidden
+ ),
+ "layer.down_proj": torch.arange(experts * hidden * intermediate, dtype=torch.float32).view(
+ experts, hidden, intermediate
+ ),
+ }
+ reader = SimpleNamespace(
+ tensors=banks,
+ bypass_page_cache=False,
+ shape=lambda name: tuple(banks[name].shape),
+ read=lambda name, expert=None: banks[name][expert].clone(),
+ read_bytes=0,
+ read_calls=0,
+ )
+ reader.read_expert = lambda prefix, expert: SafetensorsReader.read_expert(reader, prefix, expert)
+ expert_bytes = 3 * intermediate * hidden * 4
+ store = ExpertStore(reader, cache_experts * expert_bytes, "cpu", torch.float32, policy=policy)
+ store.register("layer", SimpleNamespace(num_experts=experts, intermediate_dim=intermediate, hidden_dim=hidden))
+ store.allocate()
+ return store, banks
+
+ def test_lfu_keeps_a_hot_set_where_lru_thrashes(self):
+ # Repeat a decoding cycle longer than the cache.
+ pattern = [0, 1, 2, 3, 4] * 20
+ hits = {}
+ for policy in ("lru", "lfu"):
+ store, banks = self.make_store(policy, cache_experts=3)
+ for expert in pattern:
+ gate_up, down = store.get("layer", expert)
+ torch.testing.assert_close(gate_up, banks["layer.gate_up_proj"][expert])
+ torch.testing.assert_close(down, banks["layer.down_proj"][expert])
+ hits[policy] = store.hits
+ self.assertLessEqual(store.resident_bytes, store.cache_bytes)
+ self.assertEqual(hits["lru"], 0)
+ # LFU keeps 3 of the 5 experts resident after warm-up: about 3 hits per cycle of 5.
+ self.assertGreater(hits["lfu"], 0.5 * len(pattern))
+
+ def test_lfu_follows_a_new_hot_set(self):
+ store, _ = self.make_store("lfu", cache_experts=2, experts=6)
+ store.decay_every = 8
+ for expert in [0, 1] * 20 + [4, 5] * 20:
+ store.get("layer", expert)
+ self.assertEqual({key[1] for key in store.cache}, {4, 5})
+
+ def test_cold_calls_keep_the_decode_set(self):
+ store, _ = self.make_store("lru", cache_experts=3)
+ for expert in (0, 1):
+ store.acquire("layer", expert)
+ store.cold = True # a long prefill: installs 2, hits 0 without refreshing it
+ store.acquire("layer", 2)
+ store.acquire("layer", 0)
+ store.demote("layer")
+ store.cold = False
+ store.acquire("layer", 3) # evicts the cold install, not the decode set
+ self.assertEqual({key[1] for key in store.cache}, {0, 1, 3})
diff --git a/src/transformers/integrations/experts_offload.py b/src/transformers/integrations/experts_offload.py
new file mode 100644
index 0000000000..7a7686bdc0
--- /dev/null
+++ b/src/transformers/integrations/experts_offload.py
+# Copyright 2026 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Stream MoE experts from disk through a shared device cache. Inference only, on one device."""
+
+import heapq
+import json
+import math
+import os
+import re
+import threading
+from collections import OrderedDict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import torch
+from safetensors import safe_open
+
+from ..utils import logging
+from .moe import ExpertsInterface
+
+
+try:
+ import fcntl
+except ImportError: # Windows
+ fcntl = None
+
+logger = logging.get_logger(__name__)
+
+_DTYPES = {"F32": torch.float32, "F16": torch.float16, "BF16": torch.bfloat16}
+_PACKED_DTYPES = {**_DTYPES, "U32": torch.uint32, "U8": torch.uint8}
+_QUANTIZED_FORMAT = "mlx-affine-expert-blob"
+_MLX_QUANTIZATION_KERNEL = ("kernels-community/mlx-quantization-metal-kernels", 2)
+_COUNTER_LOCK = threading.Lock()
+
+
+def _read_exact(path, offset, size, bypass_page_cache, what, out=None, fd=None):
+ """Read a byte range into ``out`` (or a new buffer), optionally bypassing or dropping the OS page cache.
+
+ ``fd`` is an open descriptor of ``path``; positional reads are thread-safe."""
+ # TODO(portability): implement Windows cache bypass; report unsupported OS/filesystem combinations.
+ data = bytearray(size) if out is None else out
+ own = fd is None
+ if own:
+ fd = os.open(path, os.O_RDONLY)
+ if bypass_page_cache and hasattr(fcntl, "F_NOCACHE"):
+ fcntl.fcntl(fd, fcntl.F_NOCACHE, 1)
+ try:
+ view = memoryview(data).cast("B")
+ done = 0
+ while done < size:
+ count = os.preadv(fd, [view[done:size]], offset + done)
+ if not count:
+ raise OSError(f"Short checkpoint read: {path}, {what}")
+ done += count
+ # TODO(portability): distinguish best-effort cache eviction from uncached reads.
+ if bypass_page_cache and hasattr(os, "posix_fadvise"):
+ os.posix_fadvise(fd, offset, size, os.POSIX_FADV_DONTNEED)
+ finally:
+ if own:
+ os.close(fd)
+ return data
+
+
+class SafetensorsReader:
+ """Validate safetensors headers and read tensors or contiguous expert slices without persistent mmap."""
+
+ def __init__(self, directory, bypass_page_cache=False, packed=False):
+ directory = Path(directory)
+ self.bypass_page_cache = bypass_page_cache
+ self.dtypes = _PACKED_DTYPES if packed else _DTYPES
+ index = directory / "model.safetensors.index.json"
+ if index.exists():
+ weight_map = json.loads(index.read_text())["weight_map"]
+ filenames = sorted(set(weight_map.values()))
+ else:
+ weight_map = None
+ filenames = ["model.safetensors"]
+ self.tensors = {}
+ self.read_bytes = 0
+ self.read_calls = 0
+ self._fds = {}
+ for filename in filenames:
+ path = (directory / filename).resolve()