File size: 3,750 Bytes
524fd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
# Standalone helper plumbing; see LICENSES and NOTICE for retained notices.

"""Device, cache, and launch helpers for the standalone inference kernels."""

import contextlib
import functools
import inspect
import os
from collections.abc import Callable
from typing import Any

import torch
import triton


FLA_CACHE_RESULTS = os.getenv("FLA_CACHE_RESULTS", "1") == "1"
SUPPORTS_AUTOTUNE_CACHE = "cache_results" in inspect.signature(triton.autotune).parameters
autotune_cache_kwargs = (
    {"cache_results": FLA_CACHE_RESULTS} if SUPPORTS_AUTOTUNE_CACHE else {}
)


@functools.lru_cache(maxsize=1)
def _backend() -> str:
    try:
        return triton.runtime.driver.active.get_current_target().backend
    except RuntimeError:
        return "cpu"


is_nvidia = _backend() == "cuda"
is_nvidia_hopper = is_nvidia and (
    "NVIDIA H" in torch.cuda.get_device_name(0)
    or torch.cuda.get_device_capability()[0] >= 9
)
is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8


@functools.lru_cache(maxsize=None)
def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:
    required = {
        "ADA": 101376,
        "AMPERE": 166912,
        "HOPPER": 232448,
    }.get(arch.upper(), 102400)
    try:
        properties = triton.runtime.driver.active.utils.get_device_properties(tensor_idx)
        return properties["max_shared_mem"] >= required
    except Exception:
        return False


def tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]:
    """Retain the source's four-entry cache keyed by input object identity."""
    cache_entries: list[tuple[tuple[Any, ...], dict[str, Any], Any]] = []
    cache_size = 4

    @functools.wraps(fn)
    def wrapper(*args: Any, **kwargs: Any) -> Any:
        nonlocal cache_entries, cache_size
        for i, entry in enumerate(cache_entries):
            last_args, last_kwargs, last_result = entry
            if len(args) == len(last_args) and len(kwargs) == len(last_kwargs):
                if all(a is b for a, b in zip(args, last_args)) and all(
                    k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items()
                ):
                    cache_entries = (
                        cache_entries[:i]
                        + cache_entries[i + 1 :]
                        + [(args, kwargs, last_result)]
                    )
                    return last_result
        result = fn(*args, **kwargs)
        if len(cache_entries) >= cache_size:
            cache_entries = cache_entries[1:]
        cache_entries.append((args, kwargs, result))
        return result

    return wrapper


def input_guard(fn: Callable[..., Any]) -> Callable[..., Any]:
    """Make tensor inputs contiguous and retain their CUDA device context."""

    @functools.wraps(fn)
    def wrapper(*args: Any, **kwargs: Any) -> Any:
        contiguous_args = (
            value.contiguous() if isinstance(value, torch.Tensor) else value
            for value in args
        )
        contiguous_kwargs = {
            key: value.contiguous() if isinstance(value, torch.Tensor) else value
            for key, value in kwargs.items()
        }
        tensor = next((value for value in args if isinstance(value, torch.Tensor)), None)
        if tensor is None:
            tensor = next(
                (value for value in kwargs.values() if isinstance(value, torch.Tensor)),
                None,
            )
        context = (
            torch.cuda.device(tensor.device.index)
            if tensor is not None and tensor.is_cuda
            else contextlib.nullcontext()
        )
        with context:
            return fn(*contiguous_args, **contiguous_kwargs)

    return wrapper