areneau's picture
Update Clairic Transformers runtime
524fd9d verified
Raw History Blame Contribute Delete
3.75 kB
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
# Standalone helper plumbing; see LICENSES and NOTICE for retained notices.
"""Device, cache, and launch helpers for the standalone inference kernels."""
import contextlib
import functools
import inspect
import os
from collections.abc import Callable
from typing import Any
import torch
import triton
FLA_CACHE_RESULTS = os.getenv("FLA_CACHE_RESULTS", "1") == "1"
SUPPORTS_AUTOTUNE_CACHE = "cache_results" in inspect.signature(triton.autotune).parameters
autotune_cache_kwargs = (
{"cache_results": FLA_CACHE_RESULTS} if SUPPORTS_AUTOTUNE_CACHE else {}
)
@functools.lru_cache(maxsize=1)
def _backend() -> str:
try:
return triton.runtime.driver.active.get_current_target().backend
except RuntimeError:
return "cpu"
is_nvidia = _backend() == "cuda"
is_nvidia_hopper = is_nvidia and (
"NVIDIA H" in torch.cuda.get_device_name(0)
or torch.cuda.get_device_capability()[0] >= 9
)
is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8
@functools.lru_cache(maxsize=None)
def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:
required = {
"ADA": 101376,
"AMPERE": 166912,
"HOPPER": 232448,
}.get(arch.upper(), 102400)
try:
properties = triton.runtime.driver.active.utils.get_device_properties(tensor_idx)
return properties["max_shared_mem"] >= required
except Exception:
return False
def tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]:
"""Retain the source's four-entry cache keyed by input object identity."""
cache_entries: list[tuple[tuple[Any, ...], dict[str, Any], Any]] = []
cache_size = 4
@functools.wraps(fn)
def wrapper(*args: Any, **kwargs: Any) -> Any:
nonlocal cache_entries, cache_size
for i, entry in enumerate(cache_entries):
last_args, last_kwargs, last_result = entry
if len(args) == len(last_args) and len(kwargs) == len(last_kwargs):
if all(a is b for a, b in zip(args, last_args)) and all(
k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items()
):
cache_entries = (
cache_entries[:i]
+ cache_entries[i + 1 :]
+ [(args, kwargs, last_result)]
)
return last_result
result = fn(*args, **kwargs)
if len(cache_entries) >= cache_size:
cache_entries = cache_entries[1:]
cache_entries.append((args, kwargs, result))
return result
return wrapper
def input_guard(fn: Callable[..., Any]) -> Callable[..., Any]:
"""Make tensor inputs contiguous and retain their CUDA device context."""
@functools.wraps(fn)
def wrapper(*args: Any, **kwargs: Any) -> Any:
contiguous_args = (
value.contiguous() if isinstance(value, torch.Tensor) else value
for value in args
)
contiguous_kwargs = {
key: value.contiguous() if isinstance(value, torch.Tensor) else value
for key, value in kwargs.items()
}
tensor = next((value for value in args if isinstance(value, torch.Tensor)), None)
if tensor is None:
tensor = next(
(value for value in kwargs.values() if isinstance(value, torch.Tensor)),
None,
)
context = (
torch.cuda.device(tensor.device.index)
if tensor is not None and tensor.is_cuda
else contextlib.nullcontext()
)
with context:
return fn(*contiguous_args, **contiguous_kwargs)
return wrapper