Uploaded using `kernel-builder`.

Browse files

Files changed (15) hide show

build/torch211-cxx11-cpu-x86_64-linux/__init__.py +77 -0
build/torch211-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so +3 -0
build/torch211-cxx11-cpu-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cpu-x86_64-linux/decode_ops/__init__.py +26 -0
build/torch211-cxx11-cpu-x86_64-linux/metadata.json +30 -0
build/torch212-cxx11-cpu-x86_64-linux/__init__.py +77 -0
build/torch212-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so +3 -0
build/torch212-cxx11-cpu-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cpu-x86_64-linux/decode_ops/__init__.py +26 -0
build/torch212-cxx11-cpu-x86_64-linux/metadata.json +30 -0
build/torch213-cxx11-cpu-x86_64-linux/__init__.py +77 -0
build/torch213-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so +3 -0
build/torch213-cxx11-cpu-x86_64-linux/_ops.py +9 -0
build/torch213-cxx11-cpu-x86_64-linux/decode_ops/__init__.py +26 -0
build/torch213-cxx11-cpu-x86_64-linux/metadata.json +30 -0

build/torch211-cxx11-cpu-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,77 @@

+"""decode-ops: fused decode-step glue for LLM inference on CPUs, aarch64-first.
+The non-matmul residue of a decode step as single NEON ops: RMSNorm with a
+fused residual add, in-place RoPE, SiLU-gate, greedy argmax, and
+deterministic temperature / top-k / top-p sampling from an explicit uniform
+draw. Companions: bitnet-cpu and quant-matmul (linears), cpu-attn
+(attention over an int8 KV cache).
+High-level API:
+    rmsnorm(x, weight, eps)
+    add_rmsnorm(residual, x, weight, eps)   residual += x, then norm (in place)
+    rope(q, k, pos, theta, interleaved)     in place
+    silu_mul(gate, up)
+    argmax(logits)
+    sample(logits, temperature, top_k, top_p, generator)
+"""
+from typing import Optional
+import torch
+from ._ops import ops
+def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
+    """out = x * weight / sqrt(mean(x^2, dim=-1) + eps), f32."""
+    orig = x.shape
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(x_flat)
+    ops.do_rmsnorm(out, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def add_rmsnorm(residual: torch.Tensor, x: torch.Tensor, weight: torch.Tensor,
+                eps: float = 1e-6) -> torch.Tensor:
+    """residual += x in place, then return rmsnorm(residual) * weight."""
+    assert residual.dtype == torch.float32 and residual.is_contiguous()
+    orig = residual.shape
+    r_flat = residual.view(-1, orig[-1])
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(r_flat)
+    ops.do_add_rmsnorm(out, r_flat, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def rope(q: torch.Tensor, k: torch.Tensor, pos: int, theta: float = 10000.0,
+         interleaved: bool = False) -> None:
+    """Rotate q [H, D] and k [Hkv, D] in place for one position."""
+    ops.do_rope(q, k, pos, theta, interleaved)
+def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
+    """out = silu(gate) * up, f32."""
+    g = gate.to(torch.float32).contiguous()
+    u = up.to(torch.float32).contiguous()
+    out = torch.empty_like(g)
+    ops.do_silu_mul(out, g, u)
+    return out
+def argmax(logits: torch.Tensor) -> int:
+    """Greedy token: first index of the maximum."""
+    return int(ops.do_argmax(logits.to(torch.float32).contiguous()))
+def sample(logits: torch.Tensor, temperature: float = 1.0, top_k: int = 0,
+           top_p: float = 1.0, generator: Optional[torch.Generator] = None) -> int:
+    """Deterministic sampling given the generator state.
+    HF warper order: temperature, top-k, then top-p over the renormalized
+    survivors; inverse CDF at one uniform draw. temperature <= 0 is greedy.
+    """
+    u = float(torch.rand(1, generator=generator).item())
+    return int(ops.do_sample(logits.to(torch.float32).contiguous(),
+                             temperature, top_k, top_p, u))
+__all__ = ["rmsnorm", "add_rmsnorm", "rope", "silu_mul", "argmax", "sample"]

build/torch211-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:fb8aa5aadc7b44ff71835c9995ca58d1310da456a349fc232a52265b48fdfbae
+size 126696

build/torch211-cxx11-cpu-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _decode_ops_cpu_8c2646d
+ops = torch.ops._decode_ops_cpu_8c2646d
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_decode_ops_cpu_8c2646d::{op_name}"

build/torch211-cxx11-cpu-x86_64-linux/decode_ops/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cpu-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "decode-ops",
+  "id": "_decode_ops_cpu_8c2646d",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cpu"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "MN0BI7XRM8J/dKPXlvcpR6hCSFo6MSaPwuXe9tJXKYg=",
+      "_decode_ops_cpu_8c2646d.abi3.so": "+4qlqtx7RP9xg1yZlcpY0TENpFajSfwjKlImW0j9+64=",
+      "_ops.py": "gqDMeOOEyadaj1GoIyuOqr79beK7adRFQfWU7iLMaLI=",
+      "decode_ops/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "8c2646dfb2cb907a897461070385e5df56f8c57e",
+      "dirty": false
+    }
+  }
+}

build/torch212-cxx11-cpu-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,77 @@

+"""decode-ops: fused decode-step glue for LLM inference on CPUs, aarch64-first.
+The non-matmul residue of a decode step as single NEON ops: RMSNorm with a
+fused residual add, in-place RoPE, SiLU-gate, greedy argmax, and
+deterministic temperature / top-k / top-p sampling from an explicit uniform
+draw. Companions: bitnet-cpu and quant-matmul (linears), cpu-attn
+(attention over an int8 KV cache).
+High-level API:
+    rmsnorm(x, weight, eps)
+    add_rmsnorm(residual, x, weight, eps)   residual += x, then norm (in place)
+    rope(q, k, pos, theta, interleaved)     in place
+    silu_mul(gate, up)
+    argmax(logits)
+    sample(logits, temperature, top_k, top_p, generator)
+"""
+from typing import Optional
+import torch
+from ._ops import ops
+def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
+    """out = x * weight / sqrt(mean(x^2, dim=-1) + eps), f32."""
+    orig = x.shape
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(x_flat)
+    ops.do_rmsnorm(out, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def add_rmsnorm(residual: torch.Tensor, x: torch.Tensor, weight: torch.Tensor,
+                eps: float = 1e-6) -> torch.Tensor:
+    """residual += x in place, then return rmsnorm(residual) * weight."""
+    assert residual.dtype == torch.float32 and residual.is_contiguous()
+    orig = residual.shape
+    r_flat = residual.view(-1, orig[-1])
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(r_flat)
+    ops.do_add_rmsnorm(out, r_flat, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def rope(q: torch.Tensor, k: torch.Tensor, pos: int, theta: float = 10000.0,
+         interleaved: bool = False) -> None:
+    """Rotate q [H, D] and k [Hkv, D] in place for one position."""
+    ops.do_rope(q, k, pos, theta, interleaved)
+def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
+    """out = silu(gate) * up, f32."""
+    g = gate.to(torch.float32).contiguous()
+    u = up.to(torch.float32).contiguous()
+    out = torch.empty_like(g)
+    ops.do_silu_mul(out, g, u)
+    return out
+def argmax(logits: torch.Tensor) -> int:
+    """Greedy token: first index of the maximum."""
+    return int(ops.do_argmax(logits.to(torch.float32).contiguous()))
+def sample(logits: torch.Tensor, temperature: float = 1.0, top_k: int = 0,
+           top_p: float = 1.0, generator: Optional[torch.Generator] = None) -> int:
+    """Deterministic sampling given the generator state.
+    HF warper order: temperature, top-k, then top-p over the renormalized
+    survivors; inverse CDF at one uniform draw. temperature <= 0 is greedy.
+    """
+    u = float(torch.rand(1, generator=generator).item())
+    return int(ops.do_sample(logits.to(torch.float32).contiguous(),
+                             temperature, top_k, top_p, u))
+__all__ = ["rmsnorm", "add_rmsnorm", "rope", "silu_mul", "argmax", "sample"]

build/torch212-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:0d37977e0a4af6dcb09c469602d119c538162daeaf1df10459ac89a2b4293b69
+size 126512

build/torch212-cxx11-cpu-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _decode_ops_cpu_8c2646d
+ops = torch.ops._decode_ops_cpu_8c2646d
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_decode_ops_cpu_8c2646d::{op_name}"

build/torch212-cxx11-cpu-x86_64-linux/decode_ops/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cpu-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "decode-ops",
+  "id": "_decode_ops_cpu_8c2646d",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cpu"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "MN0BI7XRM8J/dKPXlvcpR6hCSFo6MSaPwuXe9tJXKYg=",
+      "_decode_ops_cpu_8c2646d.abi3.so": "DTeXfgpK9tywnEaWAtEZxTgWLa6vHfEEWayJorQpO2k=",
+      "_ops.py": "gqDMeOOEyadaj1GoIyuOqr79beK7adRFQfWU7iLMaLI=",
+      "decode_ops/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "8c2646dfb2cb907a897461070385e5df56f8c57e",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cpu-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,77 @@

+"""decode-ops: fused decode-step glue for LLM inference on CPUs, aarch64-first.
+The non-matmul residue of a decode step as single NEON ops: RMSNorm with a
+fused residual add, in-place RoPE, SiLU-gate, greedy argmax, and
+deterministic temperature / top-k / top-p sampling from an explicit uniform
+draw. Companions: bitnet-cpu and quant-matmul (linears), cpu-attn
+(attention over an int8 KV cache).
+High-level API:
+    rmsnorm(x, weight, eps)
+    add_rmsnorm(residual, x, weight, eps)   residual += x, then norm (in place)
+    rope(q, k, pos, theta, interleaved)     in place
+    silu_mul(gate, up)
+    argmax(logits)
+    sample(logits, temperature, top_k, top_p, generator)
+"""
+from typing import Optional
+import torch
+from ._ops import ops
+def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
+    """out = x * weight / sqrt(mean(x^2, dim=-1) + eps), f32."""
+    orig = x.shape
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(x_flat)
+    ops.do_rmsnorm(out, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def add_rmsnorm(residual: torch.Tensor, x: torch.Tensor, weight: torch.Tensor,
+                eps: float = 1e-6) -> torch.Tensor:
+    """residual += x in place, then return rmsnorm(residual) * weight."""
+    assert residual.dtype == torch.float32 and residual.is_contiguous()
+    orig = residual.shape
+    r_flat = residual.view(-1, orig[-1])
+    x_flat = x.reshape(-1, orig[-1]).to(torch.float32).contiguous()
+    out = torch.empty_like(r_flat)
+    ops.do_add_rmsnorm(out, r_flat, x_flat, weight.to(torch.float32).contiguous(), eps)
+    return out.view(orig)
+def rope(q: torch.Tensor, k: torch.Tensor, pos: int, theta: float = 10000.0,
+         interleaved: bool = False) -> None:
+    """Rotate q [H, D] and k [Hkv, D] in place for one position."""
+    ops.do_rope(q, k, pos, theta, interleaved)
+def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
+    """out = silu(gate) * up, f32."""
+    g = gate.to(torch.float32).contiguous()
+    u = up.to(torch.float32).contiguous()
+    out = torch.empty_like(g)
+    ops.do_silu_mul(out, g, u)
+    return out
+def argmax(logits: torch.Tensor) -> int:
+    """Greedy token: first index of the maximum."""
+    return int(ops.do_argmax(logits.to(torch.float32).contiguous()))
+def sample(logits: torch.Tensor, temperature: float = 1.0, top_k: int = 0,
+           top_p: float = 1.0, generator: Optional[torch.Generator] = None) -> int:
+    """Deterministic sampling given the generator state.
+    HF warper order: temperature, top-k, then top-p over the renormalized
+    survivors; inverse CDF at one uniform draw. temperature <= 0 is greedy.
+    """
+    u = float(torch.rand(1, generator=generator).item())
+    return int(ops.do_sample(logits.to(torch.float32).contiguous(),
+                             temperature, top_k, top_p, u))
+__all__ = ["rmsnorm", "add_rmsnorm", "rope", "silu_mul", "argmax", "sample"]

build/torch213-cxx11-cpu-x86_64-linux/_decode_ops_cpu_8c2646d.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:3368481bb338a801dfd148f575a99e274b83aa5a0acbbafb4e279c3b20a196dd
+size 126512

build/torch213-cxx11-cpu-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _decode_ops_cpu_8c2646d
+ops = torch.ops._decode_ops_cpu_8c2646d
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_decode_ops_cpu_8c2646d::{op_name}"

build/torch213-cxx11-cpu-x86_64-linux/decode_ops/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch213-cxx11-cpu-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,30 @@

+{
+  "name": "decode-ops",
+  "id": "_decode_ops_cpu_8c2646d",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cpu"
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "MN0BI7XRM8J/dKPXlvcpR6hCSFo6MSaPwuXe9tJXKYg=",
+      "_decode_ops_cpu_8c2646d.abi3.so": "M2hIG7M4qAHf0Uj1dameJ0uDqloKy7r7TiecOyChlt0=",
+      "_ops.py": "gqDMeOOEyadaj1GoIyuOqr79beK7adRFQfWU7iLMaLI=",
+      "decode_ops/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "8c2646dfb2cb907a897461070385e5df56f8c57e",
+      "dirty": false
+    }
+  }
+}