# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang # Standalone helper plumbing; see LICENSES and NOTICE for retained notices. """Device, cache, and launch helpers for the standalone inference kernels.""" import contextlib import functools import inspect import os from collections.abc import Callable from typing import Any import torch import triton FLA_CACHE_RESULTS = os.getenv("FLA_CACHE_RESULTS", "1") == "1" SUPPORTS_AUTOTUNE_CACHE = "cache_results" in inspect.signature(triton.autotune).parameters autotune_cache_kwargs = ( {"cache_results": FLA_CACHE_RESULTS} if SUPPORTS_AUTOTUNE_CACHE else {} ) @functools.lru_cache(maxsize=1) def _backend() -> str: try: return triton.runtime.driver.active.get_current_target().backend except RuntimeError: return "cpu" is_nvidia = _backend() == "cuda" is_nvidia_hopper = is_nvidia and ( "NVIDIA H" in torch.cuda.get_device_name(0) or torch.cuda.get_device_capability()[0] >= 9 ) is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8 @functools.lru_cache(maxsize=None) def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool: required = { "ADA": 101376, "AMPERE": 166912, "HOPPER": 232448, }.get(arch.upper(), 102400) try: properties = triton.runtime.driver.active.utils.get_device_properties(tensor_idx) return properties["max_shared_mem"] >= required except Exception: return False def tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]: """Retain the source's four-entry cache keyed by input object identity.""" cache_entries: list[tuple[tuple[Any, ...], dict[str, Any], Any]] = [] cache_size = 4 @functools.wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: nonlocal cache_entries, cache_size for i, entry in enumerate(cache_entries): last_args, last_kwargs, last_result = entry if len(args) == len(last_args) and len(kwargs) == len(last_kwargs): if all(a is b for a, b in zip(args, last_args)) and all( k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items() ): cache_entries = ( cache_entries[:i] + cache_entries[i + 1 :] + [(args, kwargs, last_result)] ) return last_result result = fn(*args, **kwargs) if len(cache_entries) >= cache_size: cache_entries = cache_entries[1:] cache_entries.append((args, kwargs, result)) return result return wrapper def input_guard(fn: Callable[..., Any]) -> Callable[..., Any]: """Make tensor inputs contiguous and retain their CUDA device context.""" @functools.wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: contiguous_args = ( value.contiguous() if isinstance(value, torch.Tensor) else value for value in args ) contiguous_kwargs = { key: value.contiguous() if isinstance(value, torch.Tensor) else value for key, value in kwargs.items() } tensor = next((value for value in args if isinstance(value, torch.Tensor)), None) if tensor is None: tensor = next( (value for value in kwargs.values() if isinstance(value, torch.Tensor)), None, ) context = ( torch.cuda.device(tensor.device.index) if tensor is not None and tensor.is_cuda else contextlib.nullcontext() ) with context: return fn(*contiguous_args, **contiguous_kwargs) return wrapper