Upload 12 files

d03a3e6 verified 8 months ago

8.53 kB

"""
Copyright (c) 2025 by SpargeAttn team.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
from pathlib import Path
import subprocess
from packaging.version import parse, Version
from typing import List, Set
import warnings

from setuptools import setup, find_packages
import torch
from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME

HAS_SM90 = False
SAGE2PP_ENABLED = True

def run_instantiations(src_dir: str):
    base_path = Path(src_dir)
    py_files = [
        path for path in base_path.rglob('*.py')
        if path.is_file()
    ]

    for py_file in py_files:
        print(f"Running: {py_file}")
        os.system(f"python {py_file}")

def get_instantiations(src_dir: str):
    # get all .cu files under src_dir
    base_path = Path(src_dir)
    return [
        os.path.join(src_dir, str(path.relative_to(base_path)))
        for path in base_path.rglob('*')
        if path.is_file() and path.suffix == ".cu"
    ]

# Supported NVIDIA GPU architectures.
SUPPORTED_ARCHS = {"8.0", "8.6", "8.7", "8.9", "9.0"}

# Compiler flags.
CXX_FLAGS = ["-g", "-O3", "-fopenmp", "-lgomp", "-std=c++17", "-DENABLE_BF16"]
NVCC_FLAGS = [
    "-O3",
    "-std=c++17",
    "-U__CUDA_NO_HALF_OPERATORS__",
    "-U__CUDA_NO_HALF_CONVERSIONS__",
    "--use_fast_math",
    "--threads=8",
    "-Xptxas=-v",
    "-diag-suppress=174", # suppress the specific warning
    # "-Xcompiler", "-include,cassert", # fix error occurs when compiling for SM90+ with newer CUDA toolkits RTX 40 系列是 SM89可以安全移除该选项
]

ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]

if CUDA_HOME is None:
    raise RuntimeError(
        "Cannot find CUDA_HOME. CUDA must be available to build the package.")

def get_nvcc_cuda_version(cuda_dir: str) -> Version:
    """Get the CUDA version from nvcc.
    Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
    """
    nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
                                          universal_newlines=True)
    output = nvcc_output.split()
    release_idx = output.index("release") + 1
    nvcc_cuda_version = parse(output[release_idx].split(",")[0])
    return nvcc_cuda_version

def get_torch_arch_list() -> Set[str]:
    # TORCH_CUDA_ARCH_LIST can have one or more architectures,
    # e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
    # compiler to additionally include PTX code that can be runtime-compiled
    # and executed on the 8.6 or newer architectures. While the PTX code will
    # not give the best performance on the newer architectures, it provides
    # forward compatibility.
    env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
    if env_arch_list is None:
        return set()

    # List are separated by ; or space.
    torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
    if not torch_arch_list:
        return set()

    # Filter out the invalid architectures and print a warning.
    valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
    arch_list = torch_arch_list.intersection(valid_archs)
    # If none of the specified architectures are valid, raise an error.
    if not arch_list:
        raise RuntimeError(
            "None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
            f"variable ({env_arch_list}) is supported. "
            f"Supported CUDA architectures are: {valid_archs}.")
    invalid_arch_list = torch_arch_list - valid_archs
    if invalid_arch_list:
        warnings.warn(
            f"Unsupported CUDA architectures ({invalid_arch_list}) are "
            "excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
            f"({env_arch_list}). Supported CUDA architectures are: "
            f"{valid_archs}.")
    return arch_list

# First, check the TORCH_CUDA_ARCH_LIST environment variable.
compute_capabilities = get_torch_arch_list()
if not compute_capabilities:
    # If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
    # GPUs on the current machine.
    device_count = torch.cuda.device_count()
    for i in range(device_count):
        major, minor = torch.cuda.get_device_capability(i)
        if major < 8:
            raise RuntimeError(
                "GPUs with compute capability below 8.0 are not supported.")
        compute_capabilities.add(f"{major}.{minor}")

nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
if not compute_capabilities:
    raise RuntimeError("No GPUs found. Please specify the target GPU architectures or build on a machine with GPUs.")

# Validate the NVCC CUDA version.
if nvcc_cuda_version < Version("12.0"):
    raise RuntimeError("CUDA 12.0 or higher is required to build the package.")
if nvcc_cuda_version < Version("12.4"):
    if any(cc.startswith("8.9") for cc in compute_capabilities):
        raise RuntimeError(
            "CUDA 12.4 or higher is required for compute capability 8.9.")
    if any(cc.startswith("9.0") for cc in compute_capabilities):
        raise RuntimeError(
            "CUDA 12.4 or higher is required for compute capability 9.0.")
if nvcc_cuda_version < Version("12.8"):
    warnings.warn("CUDA 12.8 or higher is required for Sage2++")
    SAGE2PP_ENABLED = False

# Add target compute capabilities to NVCC flags.
for capability in compute_capabilities:
    num = capability.replace(".", "")
    if num == '90':
        num = '90a'
        HAS_SM90 = True
        CXX_FLAGS += ["-DHAS_SM90"]
    if num == '80' or num == '86' or num == '87':
        SAGE2PP_ENABLED = False
    
    NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
    if capability.endswith("+PTX"):
        NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]

if SAGE2PP_ENABLED:
    CXX_FLAGS += ["-DSAGE2PP_ENABLED"]

ext_modules = []

run_instantiations("csrc/qattn/instantiations_sm80")
run_instantiations("csrc/qattn/instantiations_sm89")
run_instantiations("csrc/qattn/instantiations_sm90")

sources = [
    "csrc/qattn/pybind.cpp",
    "csrc/qattn/qk_int_sv_f16_cuda_sm80.cu",
    "csrc/qattn/qk_int_sv_f8_cuda_sm89.cu",
] + get_instantiations("csrc/qattn/instantiations_sm80") + get_instantiations("csrc/qattn/instantiations_sm89")

if HAS_SM90:
    sources += ["csrc/qattn/qk_int_sv_f8_cuda_sm90.cu", ]
    sources += get_instantiations("csrc/qattn/instantiations_sm90")

qattn_extension = CUDAExtension(
    name="spas_sage_attn._qattn",
    sources=sources,
    extra_compile_args={
        "cxx": CXX_FLAGS,
        "nvcc": NVCC_FLAGS,
    },
    extra_link_args=['-lcuda'],
)
ext_modules.append(qattn_extension)

fused_extension = CUDAExtension(
    name="spas_sage_attn._fused",
    sources=["csrc/fused/pybind.cpp", "csrc/fused/fused.cu"],
    extra_compile_args={
        "cxx": CXX_FLAGS,
        "nvcc": NVCC_FLAGS,
    },
)
ext_modules.append(fused_extension)

setup(
    name='spas_sage_attn', 
    version='0.1.0',  
    author='Jintao Zhang, Chendong Xiang, Haofeng Huang',  
    author_email='jt-zhang6@gmail.com', 
    packages=find_packages(),  
    description='Accurate and efficient Sparse SageAttention.',  
    long_description=open('README.md', encoding='utf-8').read(),  
    long_description_content_type='text/markdown', 
    url='https://github.com/thu-ml/SpargeAttn', 
    license='BSD 3-Clause License', 
    python_requires='>=3.9', 
    classifiers=[  
        'Development Status :: 3 - Alpha', 
        'Intended Audience :: Developers',  
        'Topic :: Software Development :: Libraries :: Python Modules',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python :: 3', 
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
        'Programming Language :: Python :: 3.11',
        'Operating System :: OS Independent',
    ],
    ext_modules=ext_modules,
    cmdclass={"build_ext": BuildExtension},
)

	"""
	Copyright (c) 2025 by SpargeAttn team.

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	"""

	import os
	from pathlib import Path
	import subprocess
	from packaging.version import parse, Version
	from typing import List, Set
	import warnings

	from setuptools import setup, find_packages
	import torch
	from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME

	HAS_SM90 = False
	SAGE2PP_ENABLED = True

	def run_instantiations(src_dir: str):
	base_path = Path(src_dir)
	py_files = [
	path for path in base_path.rglob('*.py')
	if path.is_file()
	]

	for py_file in py_files:
	print(f"Running: {py_file}")
	os.system(f"python {py_file}")

	def get_instantiations(src_dir: str):
	# get all .cu files under src_dir
	base_path = Path(src_dir)
	return [
	os.path.join(src_dir, str(path.relative_to(base_path)))
	for path in base_path.rglob('*')
	if path.is_file() and path.suffix == ".cu"
	]

	# Supported NVIDIA GPU architectures.
	SUPPORTED_ARCHS = {"8.0", "8.6", "8.7", "8.9", "9.0"}

	# Compiler flags.
	CXX_FLAGS = ["-g", "-O3", "-fopenmp", "-lgomp", "-std=c++17", "-DENABLE_BF16"]
	NVCC_FLAGS = [
	"-O3",
	"-std=c++17",
	"-U__CUDA_NO_HALF_OPERATORS__",
	"-U__CUDA_NO_HALF_CONVERSIONS__",
	"--use_fast_math",
	"--threads=8",
	"-Xptxas=-v",
	"-diag-suppress=174", # suppress the specific warning
	# "-Xcompiler", "-include,cassert", # fix error occurs when compiling for SM90+ with newer CUDA toolkits RTX 40 系列是 SM89可以安全移除该选项
	]

	ABI = 1 if torch._C._GLIBCXX_USE_CXX11_ABI else 0
	CXX_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]
	NVCC_FLAGS += [f"-D_GLIBCXX_USE_CXX11_ABI={ABI}"]

	if CUDA_HOME is None:
	raise RuntimeError(
	"Cannot find CUDA_HOME. CUDA must be available to build the package.")

	def get_nvcc_cuda_version(cuda_dir: str) -> Version:
	"""Get the CUDA version from nvcc.

	Adapted from https://github.com/NVIDIA/apex/blob/8b7a1ff183741dd8f9b87e7bafd04cfde99cea28/setup.py
	"""
	nvcc_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
	universal_newlines=True)
	output = nvcc_output.split()
	release_idx = output.index("release") + 1
	nvcc_cuda_version = parse(output[release_idx].split(",")[0])
	return nvcc_cuda_version

	def get_torch_arch_list() -> Set[str]:
	# TORCH_CUDA_ARCH_LIST can have one or more architectures,
	# e.g. "8.0" or "7.5,8.0,8.6+PTX". Here, the "8.6+PTX" option asks the
	# compiler to additionally include PTX code that can be runtime-compiled
	# and executed on the 8.6 or newer architectures. While the PTX code will
	# not give the best performance on the newer architectures, it provides
	# forward compatibility.
	env_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None)
	if env_arch_list is None:
	return set()

	# List are separated by ; or space.
	torch_arch_list = set(env_arch_list.replace(" ", ";").split(";"))
	if not torch_arch_list:
	return set()

	# Filter out the invalid architectures and print a warning.
	valid_archs = SUPPORTED_ARCHS.union({s + "+PTX" for s in SUPPORTED_ARCHS})
	arch_list = torch_arch_list.intersection(valid_archs)
	# If none of the specified architectures are valid, raise an error.
	if not arch_list:
	raise RuntimeError(
	"None of the CUDA architectures in `TORCH_CUDA_ARCH_LIST` env "
	f"variable ({env_arch_list}) is supported. "
	f"Supported CUDA architectures are: {valid_archs}.")
	invalid_arch_list = torch_arch_list - valid_archs
	if invalid_arch_list:
	warnings.warn(
	f"Unsupported CUDA architectures ({invalid_arch_list}) are "
	"excluded from the `TORCH_CUDA_ARCH_LIST` env variable "
	f"({env_arch_list}). Supported CUDA architectures are: "
	f"{valid_archs}.")
	return arch_list

	# First, check the TORCH_CUDA_ARCH_LIST environment variable.
	compute_capabilities = get_torch_arch_list()
	if not compute_capabilities:
	# If TORCH_CUDA_ARCH_LIST is not defined or empty, target all available
	# GPUs on the current machine.
	device_count = torch.cuda.device_count()
	for i in range(device_count):
	major, minor = torch.cuda.get_device_capability(i)
	if major < 8:
	raise RuntimeError(
	"GPUs with compute capability below 8.0 are not supported.")
	compute_capabilities.add(f"{major}.{minor}")

	nvcc_cuda_version = get_nvcc_cuda_version(CUDA_HOME)
	if not compute_capabilities:
	raise RuntimeError("No GPUs found. Please specify the target GPU architectures or build on a machine with GPUs.")

	# Validate the NVCC CUDA version.
	if nvcc_cuda_version < Version("12.0"):
	raise RuntimeError("CUDA 12.0 or higher is required to build the package.")
	if nvcc_cuda_version < Version("12.4"):
	if any(cc.startswith("8.9") for cc in compute_capabilities):
	raise RuntimeError(
	"CUDA 12.4 or higher is required for compute capability 8.9.")
	if any(cc.startswith("9.0") for cc in compute_capabilities):
	raise RuntimeError(
	"CUDA 12.4 or higher is required for compute capability 9.0.")
	if nvcc_cuda_version < Version("12.8"):
	warnings.warn("CUDA 12.8 or higher is required for Sage2++")
	SAGE2PP_ENABLED = False

	# Add target compute capabilities to NVCC flags.
	for capability in compute_capabilities:
	num = capability.replace(".", "")
	if num == '90':
	num = '90a'
	HAS_SM90 = True
	CXX_FLAGS += ["-DHAS_SM90"]
	if num == '80' or num == '86' or num == '87':
	SAGE2PP_ENABLED = False

	NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=sm_{num}"]
	if capability.endswith("+PTX"):
	NVCC_FLAGS += ["-gencode", f"arch=compute_{num},code=compute_{num}"]

	if SAGE2PP_ENABLED:
	CXX_FLAGS += ["-DSAGE2PP_ENABLED"]

	ext_modules = []

	run_instantiations("csrc/qattn/instantiations_sm80")
	run_instantiations("csrc/qattn/instantiations_sm89")
	run_instantiations("csrc/qattn/instantiations_sm90")

	sources = [
	"csrc/qattn/pybind.cpp",
	"csrc/qattn/qk_int_sv_f16_cuda_sm80.cu",
	"csrc/qattn/qk_int_sv_f8_cuda_sm89.cu",
	] + get_instantiations("csrc/qattn/instantiations_sm80") + get_instantiations("csrc/qattn/instantiations_sm89")

	if HAS_SM90:
	sources += ["csrc/qattn/qk_int_sv_f8_cuda_sm90.cu", ]
	sources += get_instantiations("csrc/qattn/instantiations_sm90")

	qattn_extension = CUDAExtension(
	name="spas_sage_attn._qattn",
	sources=sources,
	extra_compile_args={
	"cxx": CXX_FLAGS,
	"nvcc": NVCC_FLAGS,
	},
	extra_link_args=['-lcuda'],
	)
	ext_modules.append(qattn_extension)

	fused_extension = CUDAExtension(
	name="spas_sage_attn._fused",
	sources=["csrc/fused/pybind.cpp", "csrc/fused/fused.cu"],
	extra_compile_args={
	"cxx": CXX_FLAGS,
	"nvcc": NVCC_FLAGS,
	},
	)
	ext_modules.append(fused_extension)

	setup(
	name='spas_sage_attn',
	version='0.1.0',
	author='Jintao Zhang, Chendong Xiang, Haofeng Huang',
	author_email='jt-zhang6@gmail.com',
	packages=find_packages(),
	description='Accurate and efficient Sparse SageAttention.',
	long_description=open('README.md', encoding='utf-8').read(),
	long_description_content_type='text/markdown',
	url='https://github.com/thu-ml/SpargeAttn',
	license='BSD 3-Clause License',
	python_requires='>=3.9',
	classifiers=[
	'Development Status :: 3 - Alpha',
	'Intended Audience :: Developers',
	'Topic :: Software Development :: Libraries :: Python Modules',
	'License :: OSI Approved :: BSD License',
	'Programming Language :: Python :: 3',
	'Programming Language :: Python :: 3.9',
	'Programming Language :: Python :: 3.10',
	'Programming Language :: Python :: 3.11',
	'Operating System :: OS Independent',
	],
	ext_modules=ext_modules,
	cmdclass={"build_ext": BuildExtension},
	)