diff --git a/build/torch210-cxx11-cu126-x86_64-linux/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch210-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch210-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 44062d854ae6efd15a5d62c06bfb87da75dd5b88..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e964b58e4e1d4008e7a2431a9f248d8402a9a65fa10b2a0257c746235e4e289e -size 3126824 diff --git a/build/torch210-cxx11-cu126-x86_64-linux/_ops.py b/build/torch210-cxx11-cu126-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu126-x86_64-linux/activation/__init__.py b/build/torch210-cxx11-cu126-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch210-cxx11-cu126-x86_64-linux/layers.py b/build/torch210-cxx11-cu126-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch210-cxx11-cu126-x86_64-linux/metadata.json b/build/torch210-cxx11-cu126-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu126-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch210-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch210-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 2fced462840f1e091cd7d3142af3fb608a1ce602..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b4b96fdb577e1f109b2ae39eb9b546a6ff48b71a22bc3d008fef4c36e8c338aa -size 4406608 diff --git a/build/torch210-cxx11-cu128-x86_64-linux/_ops.py b/build/torch210-cxx11-cu128-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu128-x86_64-linux/activation/__init__.py b/build/torch210-cxx11-cu128-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch210-cxx11-cu128-x86_64-linux/layers.py b/build/torch210-cxx11-cu128-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch210-cxx11-cu128-x86_64-linux/metadata.json b/build/torch210-cxx11-cu128-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu128-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch210-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch210-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 21f96542b3d60ba0650d92dbc795d6a050e195ab..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c688c1e1ba5c179198199a538379a9692a19cab64d67102bf7faa1b15afa55fe -size 4190152 diff --git a/build/torch210-cxx11-cu130-x86_64-linux/_ops.py b/build/torch210-cxx11-cu130-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch210-cxx11-cu130-x86_64-linux/activation/__init__.py b/build/torch210-cxx11-cu130-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch210-cxx11-cu130-x86_64-linux/layers.py b/build/torch210-cxx11-cu130-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch210-cxx11-cu130-x86_64-linux/metadata.json b/build/torch210-cxx11-cu130-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch210-cxx11-cu130-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch28-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch28-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 18cf7c8ab7130e7baaebb4af1ed1f96bd7dbab74..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:911660bed858a039ef77a306d7d9860f42b8bdd88ba3c286e93ed6e4e29de10f -size 3121056 diff --git a/build/torch28-cxx11-cu126-x86_64-linux/_ops.py b/build/torch28-cxx11-cu126-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu126-x86_64-linux/activation/__init__.py b/build/torch28-cxx11-cu126-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu126-x86_64-linux/layers.py b/build/torch28-cxx11-cu126-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch28-cxx11-cu126-x86_64-linux/metadata.json b/build/torch28-cxx11-cu126-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu126-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch28-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index a8afe27bc31f59f4962bcb3e5b480846f75cdb98..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7624549b7daad396ef5c550d3aac9b511fce95ba560f4cff9560f255b6212895 -size 4400792 diff --git a/build/torch28-cxx11-cu128-x86_64-linux/_ops.py b/build/torch28-cxx11-cu128-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu128-x86_64-linux/activation/__init__.py b/build/torch28-cxx11-cu128-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu128-x86_64-linux/layers.py b/build/torch28-cxx11-cu128-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch28-cxx11-cu128-x86_64-linux/metadata.json b/build/torch28-cxx11-cu128-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu128-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch28-cxx11-cu129-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 6553a6d625d94c5bc54edd8ed0ab27bb59614570..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:59858d94c625ceae30130a9e481971d577e69acb5fe1389610d299519d55a5d3 -size 4438672 diff --git a/build/torch28-cxx11-cu129-x86_64-linux/_ops.py b/build/torch28-cxx11-cu129-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch28-cxx11-cu129-x86_64-linux/activation/__init__.py b/build/torch28-cxx11-cu129-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch28-cxx11-cu129-x86_64-linux/layers.py b/build/torch28-cxx11-cu129-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch28-cxx11-cu129-x86_64-linux/metadata.json b/build/torch28-cxx11-cu129-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch28-cxx11-cu129-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch29-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 3afa89ecf87a56b7e0a18d278b5dee26cdd6bf41..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:89c27579202d7bb91fe260ba72dc77634d48662859ce719632f756e5a5a3d8eb -size 3121128 diff --git a/build/torch29-cxx11-cu126-x86_64-linux/_ops.py b/build/torch29-cxx11-cu126-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu126-x86_64-linux/activation/__init__.py b/build/torch29-cxx11-cu126-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu126-x86_64-linux/layers.py b/build/torch29-cxx11-cu126-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch29-cxx11-cu126-x86_64-linux/metadata.json b/build/torch29-cxx11-cu126-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu126-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch29-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 9c636e9d01197e8cd61d8fa0674b05486089c991..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bf82685c2a9d82d9fee85dc71b12c6ae0130134757bf9af801fcf624d92d25c4 -size 4400864 diff --git a/build/torch29-cxx11-cu128-x86_64-linux/_ops.py b/build/torch29-cxx11-cu128-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu128-x86_64-linux/activation/__init__.py b/build/torch29-cxx11-cu128-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu128-x86_64-linux/layers.py b/build/torch29-cxx11-cu128-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch29-cxx11-cu128-x86_64-linux/metadata.json b/build/torch29-cxx11-cu128-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu128-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/__init__.py deleted file mode 100644 index 1a9cd15a0a75f95c5ab956fb05c2a9860f218156..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import torch - -from ._ops import ops - -from . import layers - - -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu_and_mul(out, x) - return out - - -def mul_and_silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.mul_and_silu(out, x) - return out - - -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_and_mul(out, x) - return out - - -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh_and_mul(out, x) - return out - - -def fatrelu_and_mul(out: torch.Tensor, x: torch.Tensor, threshold: float = 0.0) -> None: - ops.fatrelu_and_mul(out, x, threshold) - return out - - -def gelu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu(out, x) - return out - -def silu(out: torch.Tensor, x: torch.Tensor) -> None: - ops.silu(out, x) - return out - - -def gelu_tanh(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_tanh(out, x) - return out - - -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_fast(out, x) - return out - - -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_new(out, x) - return out - - -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None: - ops.gelu_quick(out, x) - return out - - -__all__ = [ - "silu_and_mul", - "mul_and_silu", - "gelu_and_mul", - "gelu_tanh_and_mul", - "fatrelu_and_mul", - "gelu_fast", - "gelu_new", - "gelu_quick", - "gelu_tanh", - "silu", - "gelu", - "layers", -] diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so b/build/torch29-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so deleted file mode 100644 index 1ce60c683668e58de7b910c09c913982bbcfb84a..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/_activation_d8d5fa1.abi3.so +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2859f0a3d4cacf1c12aa748fc12f1a3b92ad40654d00b15b259f1f13e9d349fd -size 4180240 diff --git a/build/torch29-cxx11-cu130-x86_64-linux/_ops.py b/build/torch29-cxx11-cu130-x86_64-linux/_ops.py deleted file mode 100644 index ad2904963baa92a46f7324881aaba6e26fef91bb..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/_ops.py +++ /dev/null @@ -1,9 +0,0 @@ -import torch -from . import _activation_d8d5fa1 -ops = torch.ops._activation_d8d5fa1 - -def add_op_namespace_prefix(op_name: str): - """ - Prefix op by namespace. - """ - return f"_activation_d8d5fa1::{op_name}" \ No newline at end of file diff --git a/build/torch29-cxx11-cu130-x86_64-linux/activation/__init__.py b/build/torch29-cxx11-cu130-x86_64-linux/activation/__init__.py deleted file mode 100644 index 03dbc1afe1cf156661a2b1b22003cd5f599a0309..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/activation/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -import ctypes -import sys - -import importlib -from pathlib import Path -from types import ModuleType - -def _import_from_path(file_path: Path) -> ModuleType: - # We cannot use the module name as-is, after adding it to `sys.modules`, - # it would also be used for other imports. So, we make a module name that - # depends on the path for it to be unique using the hex-encoded hash of - # the path. - path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) - module_name = path_hash - spec = importlib.util.spec_from_file_location(module_name, file_path) - if spec is None: - raise ImportError(f"Cannot load spec for {module_name} from {file_path}") - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError(f"Cannot load module {module_name} from spec") - sys.modules[module_name] = module - spec.loader.exec_module(module) # type: ignore - return module - - -globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch29-cxx11-cu130-x86_64-linux/layers.py b/build/torch29-cxx11-cu130-x86_64-linux/layers.py deleted file mode 100644 index 0aec9c95fa75e4d3ff699ce69fc6618798b179c1..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/layers.py +++ /dev/null @@ -1,179 +0,0 @@ -import torch -import torch.nn as nn - -from ._ops import ops - - -class SiluAndMul(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> silu(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.silu_and_mul(out, x) - return out - -class Silu(nn.Module): - """An activation function for SiLU. - - The function computes x -> silu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.silu(out, x) - return out - -class Gelu(nn.Module): - """An activation function for GELU. - - The function computes x -> gelu(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu(out, x) - return out - -class GeluTanh(nn.Module): - """An activation function for GELU with `tanh` approximation. - - The function computes x -> gelu_tanh(x). - - Shapes: - x: (num_tokens, d) or (batch_size, seq_len, d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - out = torch.empty_like(x) - ops.gelu_tanh(out, x) - return out - - -class MulAndSilu(nn.Module): - """An activation function for SwiGLU. - - The function computes x -> x[:d] * silu(x[d:]) where d = x.shape[-1] // 2. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.mul_and_silu(out, x) - return out - - -class GeluAndMul(nn.Module): - """An activation function for GeGLU. - - The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2. - - Shapes: - x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d) - return: (batch_size, seq_len, d) or (num_tokens, d) - """ - - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_and_mul(out, x) - return out - - -class GeluTanhAndMul(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.gelu_tanh_and_mul(out, x) - return out - - -class FatreluAndMul(nn.Module): - """An activation function for FATReLU. - - The function computes x -> FATReLU(x[:d]) * x[d:] where - d = x.shape[-1] // 2. - This is used in openbmb/MiniCPM-S-1B-sft. - - Shapes: - x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d) - return: (num_tokens, d) or (batch_size, seq_len, d) - """ - - can_torch_compile: bool = True - - def __init__(self, threshold: float = 0.0): - super().__init__() - self.threshold = threshold - - def forward(self, x: torch.Tensor): - d = x.shape[-1] // 2 - output_shape = x.shape[:-1] + (d,) - out = torch.empty(output_shape, dtype=x.dtype, device=x.device) - ops.fatrelu_and_mul(out, x, self.threshold) - return out - - -class FastGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_fast(out, x) - return out - - -class NewGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_new(out, x) - return out - - -class QuickGELU(nn.Module): - can_torch_compile: bool = True - - def forward(self, x: torch.Tensor) -> torch.Tensor: - out = torch.empty_like(x) - ops.gelu_quick(out, x) - return out diff --git a/build/torch29-cxx11-cu130-x86_64-linux/metadata.json b/build/torch29-cxx11-cu130-x86_64-linux/metadata.json deleted file mode 100644 index 76bafa5f33b6818aa6bb4cab04be811b87519b44..0000000000000000000000000000000000000000 --- a/build/torch29-cxx11-cu130-x86_64-linux/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"python-depends":[]} \ No newline at end of file