model.py

import functools
import math
import os
import warnings
from dataclasses import dataclass
from typing import Optional, Tuple

import torch
import torch.nn as nn
import torch.utils.checkpoint
from torch.nn import functional as F

_native_rms_norm = getattr(F, "rms_norm", None)
_HAS_NATIVE_RMSNORM = _native_rms_norm is not None
if not _HAS_NATIVE_RMSNORM:
    raise ImportError("model.py requires torch>=2.4 (F.rms_norm)")

try:
    import config as alpha_config
except ImportError:  # pragma: no cover - fallback for package-style imports
    from . import config as alpha_config

_LigerRMSNormFn = None
try:
    # pyrefly: ignore [missing-import]
    from liger_kernel.ops.rms_norm import LigerRMSNormFunction as _LigerRMSNormFn
except ImportError:
    pass

try:
    import kernel
except ImportError:  # pragma: no cover - fallback for package-style imports
    from . import kernel

try:
    import kernel_triton as _moe_kernel
except ImportError:
    try:
        from . import kernel_triton as _moe_kernel
    except ImportError:
        _moe_kernel = None
    except Exception as _e:  # noqa: BLE001 -- optional backend must not block model import
        warnings.warn(f"[model.py/moe] kernel_triton import failed ({_e!r}) -- using eager MoE loop.")
        _moe_kernel = None
except Exception as _e:  # noqa: BLE001 -- optional backend must not block model import
    warnings.warn(f"[model.py/moe] kernel_triton import failed ({_e!r}) -- using eager MoE loop.")
    _moe_kernel = None


def _moe_kernel_ready() -> bool:
    # Only when the config switch (use_triton_fusions) is on. Otherwise the eager expert loop runs,
    # so no graph-breaking kernel entry point is reached.
    return _moe_kernel is not None and bool(_moe_kernel.fusion_enabled())

# -- Optional Triton fusions (triton_kernel.py) --------------------------------
# Auto-detected, no config flag. Kill switch for debugging / A-B timing:
#   ALPHA_DISABLE_TRITON=1
# Every op is verified once against the eager reference (outputs AND grads) on a
# small slice of the first real call, then used; any failure falls back to eager
# for that (op, dtype/shape-key) and warns once. See _triton_attempt below.
# Config switch: config.py `use_triton_fusions` (default False). Off = no triton_kernel import and every
# `if _HAS_TRITON` branch is dead code under dynamo.
def _triton_fusions_on_in_config() -> bool:
    return bool(getattr(alpha_config, "use_triton_fusions", False))


_triton_kernel = None
if os.environ.get("ALPHA_DISABLE_TRITON") != "1" and _triton_fusions_on_in_config():
    try:
        import triton_kernel as _triton_kernel
    except ImportError:  # pragma: no cover - fallback for package-style imports
        try:
            from . import triton_kernel as _triton_kernel
        except ImportError:
            _triton_kernel = None
    except Exception as _e:  # noqa: BLE001 -- file present but broken: warn, run eager
        warnings.warn(f"[model.py/triton] triton_kernel failed to import ({_e!r}) -- running eager.")
        _triton_kernel = None


def _triton_ready() -> bool:
    if _triton_kernel is None:
        return False
    for _name in ("is_available", "_ok"):
        _fn = getattr(_triton_kernel, _name, None)
        if callable(_fn):
            try:
                return bool(_fn())
            except Exception:  # noqa: BLE001
                return False
    return True


# Static module constant: when False dynamo drops every Triton branch at trace
# time (no graph breaks, no overhead).
_HAS_TRITON = _triton_ready()
_TRITON_OK: set = set()
_TRITON_BAD: set = set()
_triton_warned: set = set()

try:
    from ngram import Engram, NgramHasher
except ImportError:  # pragma: no cover - fallback for package-style imports
    try:
        from .ngram import Engram, NgramHasher
    except ImportError:
        Engram = None
        NgramHasher = None



_alpha_config_warned: set[str] = set()

_TORCHAO_AVAILABLE: bool = False
_NVFP4_RHT_AVAILABLE: bool = False
_INT8_AVAILABLE: bool = False
# BUGFIX: these two were only bound inside the `try` body. If `import torchao`
# succeeded but a later import in the same try block raised, they stayed
# undefined and apply_int8_to_model() hit a NameError instead of the intended
# RuntimeError. Bind safe defaults up front.
_INT8_GROUPED_AVAILABLE: bool = False
_Int8WOConfig = None  # type: ignore[assignment]
try:
    import torchao as _torchao_probe  # noqa: F401
    _TORCHAO_AVAILABLE = True
    from torchao.quantization import quantize_ as _torchao_quantize
    from torchao.quantization import (
        Int8DynamicActivationInt8WeightConfig as _Int8Config,
    )
    
    try:
        from torchao.quantization import Int8WeightOnlyConfig as _Int8WOConfig
        _INT8_GROUPED_AVAILABLE = True
    except ImportError:
        _Int8WOConfig = None  # type: ignore[assignment]
        _INT8_GROUPED_AVAILABLE = False
    _INT8_AVAILABLE = True
    try:
        from torchao.prototype.mx_formats import (
            NVFP4DynamicActivationNVFP4WeightConfig as _NVFP4Config,
        )
        _NVFP4_RHT_AVAILABLE = True
    except ImportError:
        _NVFP4Config = None  # type: ignore[assignment]
except ImportError:
    _torchao_quantize = None  # type: ignore[assignment]
    _Int8Config = None        # type: ignore[assignment]
    _NVFP4Config = None       # type: ignore[assignment]

_NVFP4_DISABLED_REASON: "str | None" = None


_NVFP4_SKIP_KEYWORDS: tuple[str, ...] = (
    "wte", "lm_head", "ln_", "ln_f", "norm",
    "router", "gate", "expert_bias", "engram",
)


def _nvfp4_filter(module: nn.Module, fqn: str) -> bool:
    """torchao filter_fn: True = quantize this module."""
    if not isinstance(module, nn.Linear):
        return False
    for kw in _NVFP4_SKIP_KEYWORDS:
        if kw in fqn:
            return False
    
    out_f, in_f = module.weight.shape
    if in_f < 32 or out_f < 64 or in_f % 16 != 0:
        return False
    return True


def _nvfp4_enabled() -> bool:
    global _NVFP4_DISABLED_REASON
    if not getattr(alpha_config, 'use_nvfp4', False):
        return False
    if not torch.cuda.is_available():
        _NVFP4_DISABLED_REASON = "nvFP4: no CUDA device"
        return False
    major, _minor = torch.cuda.get_device_capability()
    if major < 10:
        msg = (
            f"nvFP4 requires Blackwell (sm_100+); found sm_{major}{_minor} "
            f"({torch.cuda.get_device_name()}) -- disabled."
        )
        if msg not in _alpha_config_warned:
            _alpha_config_warned.add(msg)
            warnings.warn(msg)
        _NVFP4_DISABLED_REASON = msg
        return False
    if not _NVFP4_RHT_AVAILABLE:
        _NVFP4_DISABLED_REASON = (
            "nvFP4: torchao or torchao.prototype.mx_formats not available. "
            "Install: pip install torchao --pre"
        )
        warnings.warn(_NVFP4_DISABLED_REASON)
        return False
    return True


def apply_nvfp4_to_model(model: nn.Module) -> int:
    
    before_ids = {fqn: id(m.weight) for fqn, m in model.named_modules()
                  if _nvfp4_filter(m, fqn)}
    _torchao_quantize(model, _NVFP4Config(), filter_fn=_nvfp4_filter)
    # BUGFIX: torchao swaps the *weight* for a tensor subclass and leaves the
    # module a plain nn.Linear, so the old `type(m).__name__ != "Linear"` test
    # always counted 0. Compare weight identity instead (same test the INT8
    # path already used).
    current = dict(model.named_modules())
    count = 0
    for fqn, old_id in before_ids.items():
        m = current.get(fqn)
        if m is not None and getattr(m, "weight", None) is not None and id(m.weight) != old_id:
            count += 1
    model._quant_mode = "nvfp4"
    model._use_nvfp4 = True
    model._use_int8 = False
    return count



# -- FP8 *training* (torchao.float8.Float8Linear) ----------------------------------
# Master weights stay in their own dtype and the state_dict is unchanged, so FP8-trained
# checkpoints are ordinary checkpoints (_quant_mode stays None). Applied explicitly from
# train.py (not GPT.__init__), so inference never sees Float8Linear.
_FP8_AVAILABLE: bool = False
try:
    from torchao.float8 import convert_to_float8_training as _convert_to_float8_training
    from torchao.float8 import Float8LinearConfig as _Float8LinearConfig
    _FP8_AVAILABLE = True
except Exception:  # noqa: BLE001 -- torchao missing/too old: FP8 simply unavailable
    _convert_to_float8_training = None  # type: ignore[assignment]
    _Float8LinearConfig = None          # type: ignore[assignment]

# nn.Linear layers that always stay in BF16 (note: "gate" alone would also hit SwiGLU's W_gate).
_FP8_SKIP_KEYWORDS: tuple[str, ...] = (
    "wte", "lm_head", "ln_", "ln_f", "norm", "router", "expert_bias", "engram", "attn_gate_proj", "mtp",
    "routed_experts",
)
_FP8_BLOCK_RE = __import__("re").compile(r"(?:^|\.)h\.(\d+)\.")
_FP8_RECIPES = ("tensorwise", "rowwise", "rowwise_with_gw_hp")


def _fp8_enabled() -> bool:
    """use_fp8=True and the hardware/library can do it. Resolved once per run (train.py)."""
    if not getattr(alpha_config, 'use_fp8', False):
        return False
    msg = None
    if not torch.cuda.is_available():
        msg = "FP8 training: no CUDA device -- running BF16."
    elif torch.cuda.get_device_capability() < (8, 9):
        msg = (f"FP8 training needs sm_89+ (Ada/Hopper/Blackwell); found "
               f"sm_{''.join(map(str, torch.cuda.get_device_capability()))} -- running BF16.")
    elif not _FP8_AVAILABLE:
        msg = "FP8 training: torchao.float8 not importable (pip install torchao) -- running BF16."
    if msg is not None:
        if msg not in _alpha_config_warned:
            _alpha_config_warned.add(msg)
            warnings.warn(msg)
        return False
    return True


def apply_fp8_to_model(model: nn.Module, n_layer: int, recipe: str = "rowwise_with_gw_hp",
                       bf16_tail_frac: float = 0.15, emulate: bool = False) -> dict:
    """Swap eligible nn.Linear -> torchao Float8Linear in place (must run BEFORE torch.compile).
    Keeps the last max(1, round(frac * n_layer)) blocks and everything in _FP8_SKIP_KEYWORDS in
    BF16. Eligible = no bias, both dims multiples of 16 (scaled_mm) and >= 64 (tiny GEMMs lose).
    `emulate` runs the matmul in high precision (tests on non-FP8 hardware only)."""
    import dataclasses as _dc
    if recipe not in _FP8_RECIPES:
        raise ValueError(f"fp8 recipe {recipe!r} not in {_FP8_RECIPES}")
    if not _FP8_AVAILABLE:
        raise RuntimeError("torchao.float8 not importable")
    n_tail = max(1, round(bf16_tail_frac * n_layer)) if bf16_tail_frac > 0 else 0
    first_bf16 = n_layer - n_tail
    kept_tail = [0]

    def _filter(mod: nn.Module, fqn: str) -> bool:
        if type(mod) is not nn.Linear or mod.bias is not None:
            return False
        if any(k in fqn for k in _FP8_SKIP_KEYWORDS):
            return False
        out_f, in_f = mod.weight.shape
        if in_f % 16 or out_f % 16 or in_f < 64 or out_f < 64:
            return False
        mb = _FP8_BLOCK_RE.search(fqn)
        if mb is not None and int(mb.group(1)) >= first_bf16:
            kept_tail[0] += 1
            return False
        return True

    cfg = _Float8LinearConfig.from_recipe_name(recipe)
    if emulate:
        cfg = _dc.replace(cfg, emulate=True)
    before = {fqn for fqn, m in model.named_modules() if type(m) is nn.Linear}
    _convert_to_float8_training(model, module_filter_fn=_filter, config=cfg)
    converted = sum(1 for fqn, m in model.named_modules()
                    if fqn in before and type(m).__name__ == "Float8Linear")
    model._fp8_train = converted > 0
    return {"converted": converted, "kept_bf16_tail": kept_tail[0], "tail_blocks": n_tail, "recipe": recipe}


_INT8_SKIP_KEYWORDS: tuple[str, ...] = _NVFP4_SKIP_KEYWORDS

_INT8_DISABLED_REASON: "str | None" = None


def _int8_filter(module: nn.Module, fqn: str) -> bool:
    """torchao filter_fn for INT8: True = quantize this module."""
    if not isinstance(module, nn.Linear):
        return False
    for kw in _INT8_SKIP_KEYWORDS:
        if kw in fqn:
            return False
    
    out_f, in_f = module.weight.shape
    if in_f < 64 or out_f < 32:
        return False
    return True


def _int8_enabled() -> bool:
    global _INT8_DISABLED_REASON
    if not getattr(alpha_config, 'use_int8', False):
        return False
    if not torch.cuda.is_available():
        _INT8_DISABLED_REASON = "INT8: no CUDA device"
        return False
    if not _INT8_AVAILABLE:
        _INT8_DISABLED_REASON = (
            "INT8: torchao not installed or Int8DynamicActivationInt8WeightConfig "
            "not found. Install: pip install torchao"
        )
        warnings.warn(_INT8_DISABLED_REASON)
        return False
    
    if _nvfp4_enabled():
        msg = (
            "INT8 disabled: use_nvfp4=True also set and this is SM100+ hardware -- "
            "nvFP4 takes priority. Set use_nvfp4=False to use INT8 instead."
        )
        if msg not in _alpha_config_warned:
            _alpha_config_warned.add(msg)
            warnings.warn(msg)
        _INT8_DISABLED_REASON = msg
        return False
    return True


def apply_int8_to_model(model: nn.Module, group_size: int = 128) -> int:
    """Quantize eligible Linear layers to INT8 (Jetfire-style, dynamic activations).

    Config selection priority:
      1. Int8DynamicActivationInt8WeightConfig() -- stable torchao, per-channel
         weights + dynamic per-tensor INT8 activations. Best for T4 throughput.
      2. Int8WeightOnlyConfig(group_size=N) -- weight-only, activations stay fp16.
         Only used as fallback if (1) is unavailable (shouldn't happen on stable).

    Works on T4 (sm_75+). Fused kernel emitted by torch.compile via Inductor.
    Must be called BEFORE torch.compile().
    """
    def _filter_with_gs(module: nn.Module, fqn: str) -> bool:
        if not _int8_filter(module, fqn):
            return False
        
        in_f = module.weight.shape[1]
        if in_f % 32 != 0:  # 32 = minimum alignment for any INT8 kernel
            return False
        return True

    
    # torchao replaces module.weight in place and keeps the nn.Linear object, so
    # identity must be tracked on the weight (same test apply_nvfp4_to_model uses).
    before_ids = {fqn: id(m.weight) for fqn, m in model.named_modules() if _filter_with_gs(m, fqn)}

    
    if _INT8_AVAILABLE and _Int8Config is not None:
        config_obj = _Int8Config()
    elif _INT8_GROUPED_AVAILABLE and _Int8WOConfig is not None:
        warnings.warn(
            f"[INT8] Int8DynamicActivationInt8WeightConfig unavailable -- "
            f"falling back to Int8WeightOnlyConfig(group_size={group_size}). "
            f"Activations will stay fp16 (weight-only quant).",
        )
        config_obj = _Int8WOConfig(group_size=group_size)
    else:
        raise RuntimeError("[INT8] No usable INT8 config found in torchao -- pip install torchao")

    _torchao_quantize(model, config_obj, filter_fn=_filter_with_gs)

    
    current = dict(model.named_modules())
    count = 0
    for fqn, old_id in before_ids.items():
        mod = current.get(fqn)
        if mod is not None and getattr(mod, "weight", None) is not None and id(mod.weight) != old_id:
            count += 1
    model._quant_mode = "int8"
    model._use_int8 = True
    model._use_nvfp4 = False
    return count

def _alpha_config_attr(name: str, default):
    if not hasattr(alpha_config, name):
        warnings.warn(
            f"config module has no attribute {name!r} -- falling back to default "
            f"{default!r}. If this is unexpected, check for a casing mismatch "
            "or rename in your config.py."
        )
        return default
    return getattr(alpha_config, name)

try:
    from liger_kernel.ops.rope import LigerRopeFunction as _LigerRopeFn
except ImportError:
    _LigerRopeFn = None

try:
    from liger_kernel.transformers.fused_linear_cross_entropy import (
        LigerFusedLinearCrossEntropyLoss as _LigerFusedLinearCrossEntropyLoss,
    )
except ImportError:
    _LigerFusedLinearCrossEntropyLoss = None

_liger_warned: set[str] = set()


def _liger_warn_once(key: str, msg: str) -> None:
    if key not in _liger_warned:
        _liger_warned.add(key)
        warnings.warn(f"[model.py/liger] {msg}", stacklevel=3)


def _liger_cuda_gate(x: torch.Tensor) -> bool:
    return x.device.type == "cuda"


_CAP_CACHE: dict = {}


def _liger_fused_ce_cuda_gate(x: torch.Tensor) -> bool:
    """Liger's fused linear CE backward has a dangling-buffer bug on sm_75 (T4/Turing)
    under FP16 autocast + torch.compile: grad_output is read after its CUDA buffer is
    freed, producing cudaErrorIllegalAddress in the scalar equality check at backward
    line 294. Block sm_75 specifically; sm_80+ (Ampere+) is unaffected."""
    if x.device.type != "cuda":
        return False
    if x.device not in _CAP_CACHE:
        _CAP_CACHE[x.device] = torch.cuda.get_device_capability(x.device)
    major, minor = _CAP_CACHE[x.device]
    if major < 8:  # sm_75 = Turing, sm_70 = Volta -- both affected
        return False
    return True


def _use_liger_config() -> bool:
    
    return bool(_alpha_config_attr('use_liger', False))


# Config is fixed for the run: resolve the Liger switches once, not on every forward.
_LIGER_RMSNORM_ON = _use_liger_config() and _LigerRMSNormFn is not None
_LIGER_CE_ON = _use_liger_config() and _LigerFusedLinearCrossEntropyLoss is not None


@torch.compiler.disable
def _liger_rmsnorm_attempt(x: torch.Tensor, weight: torch.Tensor, eps: float):
    
    try:
        out = _LigerRMSNormFn.apply(x, weight, eps)
        kernel._record_backend("liger_rmsnorm_success")
        return out
    except Exception as e:  # noqa: BLE001 -- must never crash training
        kernel._record_backend(f"liger_rmsnorm_fallback:{type(e).__name__}")
        _liger_warn_once(
            f"rmsnorm_fail_{type(e).__name__}",
            f"Liger RMSNorm failed on {x.device} ({e!r}) -- falling back to native/eager RMSNorm.",
        )
        return None



def _triton_warn_once(key, msg: str) -> None:
    if key not in _triton_warned:
        _triton_warned.add(key)
        warnings.warn(f"[model.py/triton] {msg}", stacklevel=3)


def triton_status() -> dict:
    """Which (op, key) pairs were verified and engaged vs fell back to eager."""
    return {"available": _HAS_TRITON, "verified": sorted(map(str, _TRITON_OK)),
            "fell_back": sorted(map(str, _TRITON_BAD))}


def _as_list(out):
    return list(out) if isinstance(out, (tuple, list)) else [out]


def _triton_close(a: torch.Tensor, b: torch.Tensor, what: str, grad: bool = False) -> None:
    dt = b.dtype
    tol = {torch.float32: 1e-3, torch.float16: 1e-2, torch.bfloat16: 3e-2}.get(dt, 3e-2)
    if grad:
        tol *= 2
    af, bf = a.float(), b.float()
    d = (af - bf).abs().max().item()
    scale = bf.abs().max().item()
    if not (d <= tol * scale + 1e-6):   # also catches NaN
        raise AssertionError(f"{what} mismatch vs eager: max|diff|={d:.3e}, ref max|.|={scale:.3e}, tol={tol}")


def _triton_probe(tri_fn, ref_fn, probe_args) -> None:
    """One-time check on a small slice: shape, dtype, values, grad flow, grad values."""
    def mk(a):
        if torch.is_tensor(a):
            b = a.detach().clone()
            if a.requires_grad and a.is_floating_point() and torch.is_grad_enabled():
                b.requires_grad_(True)
            return b
        return a
    a_t = [mk(a) for a in probe_args]
    a_r = [mk(a) for a in probe_args]
    out_t, out_r = _as_list(tri_fn(*a_t)), _as_list(ref_fn(*a_r))
    if len(out_t) != len(out_r):
        raise AssertionError(f"output arity {len(out_t)} != eager {len(out_r)}")
    for i, (x, y) in enumerate(zip(out_t, out_r)):
        if x.shape != y.shape:
            raise AssertionError(f"output[{i}] shape {tuple(x.shape)} != eager {tuple(y.shape)}")
        if x.dtype != y.dtype:
            raise AssertionError(f"output[{i}] dtype {x.dtype} != eager {y.dtype}")
        if y.is_floating_point():   # integer outputs (top-k indices / counts): shape+dtype only (ties may differ)
            _triton_close(x, y, f"output[{i}]")
    wrt = [i for i, a in enumerate(a_t) if torch.is_tensor(a) and a.requires_grad]
    if not wrt:
        return
    out_t = [o for o in out_t if o.is_floating_point()]
    out_r = [o for o in out_r if o.is_floating_point()]
    if not all(o.requires_grad for o in out_t):
        raise AssertionError("output has no grad_fn although inputs require grad (no backward?)")
    gs = []
    for y in out_r:
        n = y.numel()
        gs.append(torch.cos(torch.arange(n, device=y.device, dtype=torch.float32)).reshape(y.shape).to(y.dtype))
    g_t = torch.autograd.grad(out_t, [a_t[i] for i in wrt], gs, allow_unused=True)
    g_r = torch.autograd.grad(out_r, [a_r[i] for i in wrt], gs, allow_unused=True)
    for i, x, y in zip(wrt, g_t, g_r):
        if x is None and y is None:
            continue
        if x is None or y is None:
            raise AssertionError(f"grad[arg {i}] present on only one side (triton={x is not None})")
        _triton_close(x, y, f"grad[arg {i}]", grad=True)


@torch.compiler.disable
def _triton_attempt(op, key, tri_fn, ref_fn, args, probe_args):
    """Run tri_fn(*args) or return None (caller then runs its eager path).
    probe_args may be a callable (evaluated only on the first call for this key)."""
    tag = (op, key)
    if tag in _TRITON_BAD:
        return None
    try:
        if tag not in _TRITON_OK:
            _triton_probe(tri_fn, ref_fn, probe_args() if callable(probe_args) else probe_args)
            _TRITON_OK.add(tag)
            kernel._record_backend(f"triton_{op}_verified")
        out = tri_fn(*args)
        if torch.is_grad_enabled() and any(torch.is_tensor(a) and a.requires_grad for a in args):
            if not all(o.requires_grad for o in _as_list(out) if o.is_floating_point()):
                raise RuntimeError("output has no grad_fn although inputs require grad")
        return out
    except Exception as e:  # noqa: BLE001 -- must never crash training
        _TRITON_BAD.add(tag)
        kernel._record_backend(f"triton_{op}_fallback:{type(e).__name__}")
        _triton_warn_once(
            tag,
            f"triton {op} {key} disabled -> eager fallback ({type(e).__name__}: {e}). "
            f"If this is a signature mismatch, fix the adapter next to _tri_{op}.",
        )
        return None


# -- Adapters: the ONLY place that knows triton_kernel's call signatures --------
def _tri_rmsnorm(x, weight, eps):
    return _triton_kernel.rmsnorm(x, weight, eps)


def _tri_swiglu(gate, value):
    return _triton_kernel.swiglu(gate, value)


def _tri_rope_qk(q, k, cos, sin, rope_dim):
    return _triton_kernel.rope_qk(q, k, cos, sin, rope_dim)


def _tri_fused_ce(h, weight, t):
    return _triton_kernel.fused_linear_cross_entropy(h, weight, t, ignore_index=-1)


# -- kernel_triton fusions (attention post, QK-norm+RoPE, add+RMSNorm, router, MoE dispatch/combine, FFN) --
# Same contract as the triton_kernel ops above: verified once per (op, key) against eager on a small slice
# (outputs and grads), then used; any failure -> warn once + eager fallback. Disable all with
# ALPHA_DISABLE_TRITON=1, or per op with KT_OFF=attn_post,qk_rope,add_rmsnorm,router,route,ffn.
def _kt_on(x, op: str) -> bool:
    fn = getattr(_moe_kernel, "use_fused", None)
    return fn is not None and os.environ.get("ALPHA_DISABLE_TRITON") != "1" and bool(fn(x, op))


_KT_YDT: dict = {}


def _kt_norm_dtype(x, weight, eps):
    """dtype the eager RMSNorm path returns for (x.dtype, weight.dtype) -- learned once from a 1-row eager call."""
    key = (x.dtype, weight.dtype)
    if key not in _KT_YDT:
        _KT_YDT[key] = _rmsnorm_eager(
            torch.zeros(1, weight.numel(), dtype=x.dtype, device=x.device), weight, eps).dtype
    return _KT_YDT[key]


def _rmsnorm_eager(x, weight, eps):
    # weight cast to the input dtype: F.rms_norm falls back to a slow composite path on a dtype mismatch.
    return _native_rms_norm(x, (x.shape[-1],), weight.to(x.dtype), eps)


def _swiglu_eager(gate, value):
    return F.silu(gate) * value


class RMSNorm(nn.Module):
    """Root Mean Square Layer Normalization (used in modern transformers)"""
    def __init__(self, dim, eps=1e-5):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(dim))
        self.dim = dim

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        if _HAS_TRITON and x.is_cuda:
            out = _triton_attempt(
                "rmsnorm", (x.dtype, x.ndim, x.is_contiguous()),
                _tri_rmsnorm, _rmsnorm_eager,
                (x, self.weight, self.eps), (x[:1], self.weight, self.eps),
            )
            if out is not None:
                return out
        if _LIGER_RMSNORM_ON and x.is_cuda:
            out = _liger_rmsnorm_attempt(x, self.weight, self.eps)
            if out is not None:
                return out
        return _native_rms_norm(x, (self.dim,), self.weight.to(x.dtype), self.eps)
def rotate_half(x):
    x1, x2 = x.chunk(2, dim=-1)
    return torch.cat((-x2, x1), dim=-1)


@dataclass
class AttentionOutput:
    """Uniform return type so callers can optionally inspect weights / cache cost."""

    output: torch.Tensor
    attention_weights: Optional[torch.Tensor] = None
    kv_cache_bytes: Optional[int] = None
    selected_indices: Optional[torch.Tensor] = None


def module_output_tensor(x):
    
    return x.output if isinstance(x, AttentionOutput) else x


def reshape_heads(x: torch.Tensor, num_heads: int, head_dim: int) -> torch.Tensor:
    
    b, t, _ = x.shape
    return x.view(b, t, num_heads, head_dim).transpose(1, 2)


def merge_heads(x: torch.Tensor) -> torch.Tensor:
    
    b, h, t, d = x.shape
    return x.transpose(1, 2).contiguous().view(b, t, h * d)


def expand_kv_heads(x: torch.Tensor, num_heads: int) -> torch.Tensor:
    
    b, h_kv, t, d = x.shape
    if h_kv == num_heads:
        return x
    if num_heads % h_kv != 0:
        raise ValueError(f"num_heads ({num_heads}) must be divisible by kv heads ({h_kv})")
    reps = num_heads // h_kv
    return x.repeat_interleave(reps, dim=1)


@functools.lru_cache(maxsize=64)
def _rope_cache(seq_len: int, dim: int, device, dtype, base: float = 10000.0):
    inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim))
    t = torch.arange(seq_len, device=device, dtype=torch.float32)
    freqs = torch.outer(t, inv_freq)
    emb = torch.cat((freqs, freqs), dim=-1)
    return emb.cos().to(dtype), emb.sin().to(dtype)


def build_rope_cache(seq_len: int, dim: int, base: float = 10000.0):
    inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
    t = torch.arange(seq_len, dtype=torch.float32)
    freqs = torch.outer(t, inv_freq)
    emb = torch.cat((freqs, freqs), dim=-1)
    return emb.cos(), emb.sin()


def apply_rope(
    x: torch.Tensor,
    positions: torch.Tensor | None = None,
    rope_dim: int | None = None,
    max_position: int | None = None,
    cos_full: torch.Tensor | None = None,
    sin_full: torch.Tensor | None = None,
) -> torch.Tensor:
    
    b, h, t, d = x.shape
    rope_dim = d if rope_dim is None else rope_dim
    
    is_sequential = positions is None
    if positions is None:
        positions = torch.arange(t, device=x.device)

    if cos_full is None or sin_full is None:
        if max_position is None:
            max_position = t
        max_pos = int(max_position) if max_position is not None else int(positions.max().item()) + 1 if positions.numel() > 0 else 1
        cos_full, sin_full = _rope_cache(max_pos, rope_dim, x.device, x.dtype)

    
    if is_sequential:
        if t > cos_full.shape[0]:
            raise ValueError(
                f"RoPE table too small: highest requested position="
                f"{t - 1}, table size={cos_full.shape[0]}. If this "
                f"happened during generate(), set gen_headroom in config.py "
                f"to however many extra positions past CONTEXT (block_size) "
                f"generation needs -- see GPT.__init__'s gen_headroom "
                f"wiring. This is separate from max_gen_tokens (a hard cap "
                f"on tokens produced per generate() call, unrelated to "
                f"RoPE table size). Default gen_headroom is 0 if unset."
            )
    elif positions is not None and positions.numel() > 0:
        highest_pos = int(positions.max().item())
        if highest_pos >= cos_full.shape[0]:
            raise ValueError(
                f"RoPE table too small: highest requested position="
                f"{highest_pos}, table size={cos_full.shape[0]}. If this "
                f"happened during generate(), set gen_headroom in config.py "
                f"to however many extra positions past CONTEXT (block_size) "
                f"generation needs -- see GPT.__init__'s gen_headroom "
                f"wiring. This is separate from max_gen_tokens (a hard cap "
                f"on tokens produced per generate() call, unrelated to "
                f"RoPE table size). Default gen_headroom is 0 if unset."
            )

    
    if is_sequential:
        cos = cos_full[:t].to(x.dtype).unsqueeze(0).unsqueeze(0)
        sin = sin_full[:t].to(x.dtype).unsqueeze(0).unsqueeze(0)
    else:
        cos = cos_full[positions].to(x.dtype).unsqueeze(0).unsqueeze(0)
        sin = sin_full[positions].to(x.dtype).unsqueeze(0).unsqueeze(0)

    x_rot, x_pass = x[..., :rope_dim], x[..., rope_dim:]
    x_rot = (x_rot * cos) + (rotate_half(x_rot) * sin)
    return torch.cat([x_rot, x_pass], dim=-1) if x_pass.shape[-1] > 0 else x_rot


def _eager_rope_qk(q, k, cos, sin, rope_dim):
    return (
        apply_rope(q, positions=None, rope_dim=rope_dim, cos_full=cos, sin_full=sin),
        apply_rope(k, positions=None, rope_dim=rope_dim, cos_full=cos, sin_full=sin),
    )


def apply_rope_qk(
    q: torch.Tensor,
    k: torch.Tensor,
    cos_full: torch.Tensor,
    sin_full: torch.Tensor,
    rope_dim: int,
    positions: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
    
    b, h, t, d = q.shape
    hk = k.shape[1]

    
    if positions is None and _HAS_TRITON and q.is_cuda:
        cos_t = cos_full[:t].to(q.dtype)
        sin_t = sin_full[:t].to(q.dtype)
        result = _triton_attempt(
            "rope_qk", (q.dtype, h == hk), _tri_rope_qk, _eager_rope_qk,
            (q, k, cos_t, sin_t, rope_dim), (q[:1], k[:1], cos_t, sin_t, rope_dim),
        )
        if result is not None:
            return tuple(result)

    if (positions is None
            and _use_liger_config()
            and _LigerRopeFn is not None
            and _liger_cuda_gate(q)
            and rope_dim == d
            and h == hk):
        result = _liger_rope_attempt(q, k, cos_full[:t], sin_full[:t])
        if result is not None:
            return result
    q_rot = apply_rope(q, positions=positions, rope_dim=rope_dim, cos_full=cos_full, sin_full=sin_full)
    k_rot = apply_rope(k, positions=positions, rope_dim=rope_dim, cos_full=cos_full, sin_full=sin_full)
    return q_rot, k_rot


@torch.compiler.disable
def _liger_rope_attempt(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor):
    
    try:
        cos = cos.to(q.dtype)
        sin = sin.to(q.dtype)
        q_rot, k_rot = _LigerRopeFn.apply(q, k, cos, sin)
        kernel._record_backend("liger_rope_success")
        return q_rot, k_rot
    except Exception as e:  # noqa: BLE001 -- must never crash training
        kernel._record_backend(f"liger_rope_fallback:{type(e).__name__}")
        _liger_warn_once(
            f"rope_fail_{type(e).__name__}",
            f"Liger RoPE failed on {q.device} ({e!r}) -- falling back to eager RoPE.",
        )
        return None


@torch.compiler.disable
def _liger_fused_ce_attempt(hidden: torch.Tensor, weight: torch.Tensor, targets: torch.Tensor):
    
    try:
        loss_fn = _LigerFusedLinearCrossEntropyLoss(ignore_index=-1)
        
        # LigerFusedLinearCrossEntropyLoss.forward(lin_weight, _input, target): weight FIRST.
        loss = loss_fn(weight, hidden.reshape(-1, hidden.size(-1)), targets.reshape(-1))
        kernel._record_backend("liger_fused_ce_success")
        return loss
    except Exception as e:  # noqa: BLE001 -- must never crash training
        kernel._record_backend(f"liger_fused_ce_fallback:{type(e).__name__}")
        _liger_warn_once(
            f"fused_ce_fail_{type(e).__name__}",
            f"Liger fused linear CE failed on {hidden.device} ({e!r}) -- "
            f"falling back to eager lm_head + F.cross_entropy.",
        )
        return None


def chunked_cross_entropy(
    hidden: torch.Tensor,
    weight: torch.Tensor,
    targets: torch.Tensor,
    ignore_index: int = -1,
    chunk_size: int = 4096,
) -> torch.Tensor:
    """Triton fused linear-CE when verified, else the chunked eager path below."""
    if _HAS_TRITON and hidden.is_cuda and ignore_index == -1:
        h = hidden.reshape(-1, hidden.size(-1))
        t = targets.reshape(-1)
        valid = t != ignore_index
        if bool(valid.any()):   # one sync; the eager path syncs on n_valid too

            def _probe():
                # 192 valid + up to 64 ignored rows, so ignore_index handling is verified too.
                vi = valid.nonzero(as_tuple=True)[0][:192]
                ii = (~valid).nonzero(as_tuple=True)[0][:64]
                idx = torch.cat([vi, ii])
                return (h[idx], weight, t[idx])

            out = _triton_attempt(
                "fused_ce", (h.dtype,), _tri_fused_ce,
                lambda hh, ww, tt: _chunked_cross_entropy_eager(hh, ww, tt, ignore_index=-1),
                (h, weight, t), _probe,
            )
            if out is not None:
                return out
    return _chunked_cross_entropy_eager(hidden, weight, targets, ignore_index, chunk_size)


def _chunked_cross_entropy_eager(
    hidden: torch.Tensor,          # (B, T, D) or (N, D) already flattened
    weight: torch.Tensor,          # (vocab_size, D)
    targets: torch.Tensor,         # (B, T) or (N,) already flattened
    ignore_index: int = -1,
    chunk_size: int = 4096,
) -> torch.Tensor:
    
    h = hidden.reshape(-1, hidden.size(-1))   # (N, D)
    t = targets.reshape(-1)                    # (N,)
    N = h.size(0)
    total_loss = torch.zeros((), device=h.device, dtype=torch.float32)
    n_valid = (t != ignore_index).sum().float()
    # No data-dependent early return: an all-ignored batch still gets zero loss with a grad edge,
    # because every checkpointed chunk stays in the graph. The clamp below avoids 0/0.

    def _chunk_loss(h_chunk: torch.Tensor, t_chunk: torch.Tensor) -> torch.Tensor:
        # Recomputed during backward via checkpoint -- only the scalar loss is
        # retained, not the (chunk_size, vocab) fp32 logit tensor.
        logits_chunk = F.linear(h_chunk, weight).float()
        return F.cross_entropy(logits_chunk, t_chunk, ignore_index=ignore_index, reduction="sum")

    for start in range(0, N, chunk_size):
        end = min(start + chunk_size, N)
        h_chunk = h[start:end]                         # (C, D)
        t_chunk = t[start:end]                         # (C,)
        # checkpoint recomputes the F.linear in backward; only the scalar
        # loss_chunk is kept alive in the autograd graph between forward and
        # backward, so the (C, vocab) fp32 logit tensor is never retained.
        loss_chunk = torch.utils.checkpoint.checkpoint(
            _chunk_loss, h_chunk, t_chunk, use_reentrant=False
        )
        total_loss = total_loss + loss_chunk
    return total_loss / n_valid.clamp(min=1)


def make_causal_mask(q_len: int, k_len: int, device, q_offset: int = 0) -> torch.Tensor:
    
    q_idx = (torch.arange(q_len, device=device).view(q_len, 1) + q_offset)
    k_idx = torch.arange(k_len, device=device).view(1, k_len)
    return torch.where(k_idx <= q_idx, torch.zeros(1, device=device), torch.full((1,), float("-inf"), device=device))


def make_sliding_window_causal_mask(q_len: int, k_len: int, window_size: int, device, q_offset: int = 0) -> torch.Tensor:
    
    q_idx = (torch.arange(q_len, device=device).view(q_len, 1) + q_offset)
    k_idx = torch.arange(k_len, device=device).view(1, k_len)
    visible = (k_idx <= q_idx) & (k_idx > q_idx - window_size)
    return torch.where(visible, torch.zeros(1, device=device), torch.full((1,), float("-inf"), device=device))



def masked_softmax(scores: torch.Tensor, mask: torch.Tensor | None, dim: int = -1) -> torch.Tensor:
    if mask is not None:
        scores = scores + mask.to(scores.dtype)
    
    return torch.softmax(scores.float(), dim=dim).to(scores.dtype)


def estimate_kv_cache_bytes(
    batch_size, seq_len, num_kv_heads, head_dim, dtype,
    compression_ratio: int = 1, window_size: int | None = None,
) -> int:
   
    elem_size = torch.zeros(1, dtype=dtype).element_size()
    effective_len = max(1, -(-seq_len // compression_ratio))  # ceil div
    if window_size is not None:
        effective_len = min(effective_len, window_size)
    return int(batch_size * num_kv_heads * effective_len * head_dim * elem_size * 2)




class KVCache:
    """Per-layer K/V buffers for incremental decoding.

    Buffers are allocated lazily on first write (dtype/device/batch follow k) and
    written in place, so a decode step costs O(T) attention reads and no torch.cat.
    Invariant: after a forward, `pos` == number of tokens already cached.
    """

    def __init__(self, n_layers: int, capacity: int):
        self.capacity = int(capacity)
        self.pos = 0
        self._layers = [{"capacity": self.capacity} for _ in range(n_layers)]

    def layer(self, i: int) -> dict:
        return self._layers[i]

    def reset(self) -> None:
        self.pos = 0   # stale data past pos is never read


# -- Static architectural constants ------------------------------------------
# XSA, QK-Norm and Gated Attention are always on. No config flags: the
# checkpoint shape is fully determined by (n_embd, n_head).
#
# Attn-gate input window: how many dims of the post-norm block input the
# per-head gate reads. Smaller = fewer params. 16 is the speedrun value;
# 0 means "use full n_embd" (dense gate).
ATTN_GATE_WINDOW_DIMS = 16


class DenseMHA(nn.Module):
    

    def __init__(
        self,
        hidden_size: int,
        num_heads: int,
        head_dim: int | None = None,
        num_kv_heads: int | None = None,
        dropout: float = 0.0,
        use_rope: bool = True,
        causal: bool = True,
        bias: bool = True,
        max_seq_len: int = 4096,
        window_size: int | None = None,
        tp_size: int = 1,
        rope_dim: int | None = None,
    ) -> None:
        super().__init__()
        if head_dim is None:
            if hidden_size % num_heads != 0:
                raise ValueError("hidden_size must be divisible by num_heads when head_dim is omitted")
            head_dim = hidden_size // num_heads
        num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
        if num_heads % num_kv_heads != 0:
            raise ValueError(f"num_heads ({num_heads}) must be divisible by num_kv_heads ({num_kv_heads})")
        if tp_size > 1:
            raise NotImplementedError(
                "tensor_parallel_size > 1 is not implemented: per-rank weights are "
                "independently initialized (never sharded from a canonical weight), "
                "so DDP gradient averaging corrupts them. Set tensor_parallel_size=1. "
                "Real TP needs: init full weights once, slice per rank, and a "
                "skip_out_proj guard that honors tp_size."
            )
        if num_heads % tp_size != 0:
            raise ValueError(
                f"tensor_parallel_size ({tp_size}) must divide num_heads ({num_heads}) "
                f"for this layer -- reduce tensor_parallel_size or increase n_head."
            )

        local_num_heads = num_heads // tp_size
        
        if num_kv_heads < tp_size:
            local_num_kv_heads = num_kv_heads
        else:
            local_num_kv_heads = num_kv_heads // tp_size if num_kv_heads % tp_size == 0 else num_kv_heads

        self.hidden_size = hidden_size
        self.num_heads = local_num_heads
        self.num_kv_heads = local_num_kv_heads
        self.head_dim = head_dim
        self.inner_dim = local_num_heads * head_dim
        self.use_rope = use_rope
        self.causal = causal
        self.tp_size = tp_size
        self.tp_group = None  
        self.window_size = window_size
        
        self.rope_dim = head_dim if rope_dim is None else rope_dim

       
        
        self._q_width = self.inner_dim
        self._k_width = self.num_kv_heads * head_dim
        self._v_width = self.num_kv_heads * head_dim
        self.qkv_proj = nn.Linear(hidden_size, self._q_width + self._k_width + self._v_width, bias=bias)
        self.out_proj = nn.Linear(self.inner_dim, hidden_size, bias=bias)
        self.dropout = nn.Dropout(dropout)

        # -- Always-on QK-Norm --------------------------------------------------
        # Per-head RMSNorm over head_dim on q and k, before the rope split.
        # Bounds attention-logit scale so softmax doesn't saturate.
        self.q_norm = RMSNorm(self.head_dim, eps=1e-6)
        self.k_norm = RMSNorm(self.head_dim, eps=1e-6)

        # -- Always-on per-head XSA strength gate -------------------------------
        # tanh(alpha) in (-1, 1). Zero-init -> gate=0 -> step-0 forward is
        # numerically identical to vanilla SA.
        self.xsa_alpha = nn.Parameter(torch.zeros(self.num_heads))
        self._xsa_v_reps = local_num_heads // local_num_kv_heads

        # -- Always-on Gated Attention (Qiu et al., NeurIPS 2025) ---------------
        # Per-head sigmoid gate on the SDPA output, before XSA/out_proj. Reads
        # the first ATTN_GATE_WINDOW_DIMS dims of the post-norm block input.
        # Zero-init -> 2*sigmoid(0) = 1.0 -> identity at step 0.
        # NOTE: GPT.__init__ re-zeroes this after self.apply(_init_weights),
        # which would otherwise overwrite it with N(0, 0.02).
        self.attn_gate_window = min(ATTN_GATE_WINDOW_DIMS, hidden_size) or hidden_size
        self.attn_gate_proj = nn.Linear(self.attn_gate_window, self.num_heads, bias=False)
        nn.init.zeros_(self.attn_gate_proj.weight)

        if use_rope:
            rope_cos, rope_sin = build_rope_cache(max_seq_len, self.rope_dim)
            self.register_buffer("rope_cos", rope_cos, persistent=False)
            self.register_buffer("rope_sin", rope_sin, persistent=False)

        # XSA pos-0 guard mask: registered once at init so forward() never
        # allocates a fresh (T,) tensor.  Non-persistent: re-created from
        # max_seq_len on load, costs nothing in the state_dict.
        self.register_buffer(
            "_xsa_pos_mask",
            torch.ones(max_seq_len),
            persistent=False,
        )
        # Slot 0 is always masked off (position-0 has no context beyond itself).
        self._xsa_pos_mask[0] = 0.0

    def forward(
        self,
        hidden_states: torch.Tensor,
        output_attentions: bool = False,
        return_kv_cache_estimate: bool = False,
        skip_out_proj: bool = False,
        positions: torch.Tensor | None = None,
        kv_cache: dict | None = None,
        cache_pos: int = 0,
    ) -> torch.Tensor | AttentionOutput:
        batch_size, seq_len, _ = hidden_states.shape
        if kv_cache is not None:
            if output_attentions or positions is not None or not self.causal:
                raise ValueError("kv_cache requires causal attention, positions=None, output_attentions=False")
            if cache_pos > 0 and seq_len != 1:
                raise NotImplementedError("cached forward: one full prefill at position 0, then one token per step")
            if cache_pos + seq_len > kv_cache["capacity"]:
                raise ValueError(f"KV cache capacity {kv_cache['capacity']} exceeded at position {cache_pos + seq_len}")

        qkv = self.qkv_proj(hidden_states)
        q = None
        if self.use_rope:
            if kv_cache is not None:
                rope_c = self.rope_cos[cache_pos:cache_pos + seq_len]
                rope_s = self.rope_sin[cache_pos:cache_pos + seq_len]
            else:
                rope_c, rope_s = self.rope_cos, self.rope_sin

        # Fused QKV split + per-head QK-RMSNorm + RoPE + (B,H,T,hd) layout (+ v copy), one kernel each way.
        if self.use_rope and positions is None and qkv.is_cuda and _kt_on(qkv, "qk_rope"):
            _wq, _wk = self.q_norm.weight, self.k_norm.weight
            _eps = self.q_norm.eps
            _odt = _kt_norm_dtype(qkv, _wq, _eps)
            _rc, _rs = rope_c[:seq_len], rope_s[:seq_len]

            def _qk_ref(qkv_, wq_, wk_):
                _q, _k, _v = qkv_.split([self._q_width, self._k_width, self._v_width], dim=-1)
                _q = _rmsnorm_eager(reshape_heads(_q, self.num_heads, self.head_dim), wq_, _eps)
                _k = _rmsnorm_eager(reshape_heads(_k, self.num_kv_heads, self.head_dim), wk_, _eps)
                _q, _k = _eager_rope_qk(_q, _k, _rc, _rs, self.rope_dim)
                return _q, _k, reshape_heads(_v, self.num_kv_heads, self.head_dim)

            _r = _triton_attempt(
                "qk_rope", (qkv.dtype, self.num_heads, self.num_kv_heads, self.head_dim, self.rope_dim),
                lambda qkv_, wq_, wk_: _moe_kernel.qk_rope(
                    qkv_, wq_, wk_, _rc, _rs, self.num_heads, self.num_kv_heads, self.head_dim,
                    self.rope_dim, _eps, _odt),
                _qk_ref,
                (qkv, _wq, _wk),
                lambda: (qkv[:1],
                         torch.linspace(0.5, 1.5, self.head_dim, device=qkv.device, dtype=_wq.dtype).requires_grad_(),
                         torch.linspace(1.5, 0.5, self.head_dim, device=qkv.device, dtype=_wk.dtype).requires_grad_()),
            )
            if _r is not None:
                q, k, v = _r

        if q is None:
            q_flat, k_flat, v_flat = qkv.split([self._q_width, self._k_width, self._v_width], dim=-1)
            q = reshape_heads(q_flat, self.num_heads, self.head_dim)
            k = reshape_heads(k_flat, self.num_kv_heads, self.head_dim)
            v = reshape_heads(v_flat, self.num_kv_heads, self.head_dim)

            # QK-Norm (always on): per-head RMSNorm over head_dim, before rope.
            q = self.q_norm(q)
            k = self.k_norm(k)

            if self.use_rope:
                q, k = apply_rope_qk(
                    q,
                    k,
                    cos_full=rope_c,
                    sin_full=rope_s,
                    rope_dim=self.rope_dim,
                    positions=positions,
                )

        if kv_cache is not None:
            if "k" not in kv_cache:
                cap = kv_cache["capacity"]
                kv_cache["k"] = k.new_empty(batch_size, self.num_kv_heads, cap, self.head_dim)
                kv_cache["v"] = v.new_empty(batch_size, self.num_kv_heads, cap, self.head_dim)
            kv_cache["k"][:, :, cache_pos:cache_pos + seq_len] = k
            kv_cache["v"][:, :, cache_pos:cache_pos + seq_len] = v

        if output_attentions:
            
            if self.num_kv_heads != self.num_heads:
                reps = self.num_heads // self.num_kv_heads
                k_eager = k.repeat_interleave(reps, dim=1)
                v_eager = v.repeat_interleave(reps, dim=1)
            else:
                k_eager, v_eager = k, v
            scores = torch.matmul(q, k_eager.transpose(-2, -1)) / math.sqrt(self.head_dim)
            mask = None
            if self.window_size is not None:
                mask = make_sliding_window_causal_mask(
                    seq_len, seq_len, self.window_size, hidden_states.device
                ).view(1, 1, seq_len, seq_len)
            elif self.causal:
                mask = make_causal_mask(seq_len, seq_len, hidden_states.device).view(1, 1, seq_len, seq_len)
            weights = masked_softmax(scores, mask, dim=-1)
            
            weights = self.dropout(weights)
            context = torch.matmul(weights, v_eager)
        elif kv_cache is not None and cache_pos > 0:
            # Single-token decode: the new token sees every cached key (or the last
            # `window_size` of them for sliding-window layers), so no mask is needed.
            # GQA without copying K/V: heads of one KV group become the "query rows".
            end = cache_pos + 1
            start = 0 if self.window_size is None else max(0, end - self.window_size)
            k_all = kv_cache["k"][:, :, start:end].to(q.dtype)
            v_all = kv_cache["v"][:, :, start:end].to(q.dtype)
            reps = self.num_heads // self.num_kv_heads
            q_g = q.reshape(batch_size, self.num_kv_heads, reps, self.head_dim)
            ctx = F.scaled_dot_product_attention(q_g, k_all, v_all)
            context = ctx.reshape(batch_size, self.num_heads, 1, self.head_dim)
            weights = None
        else:
            context = kernel.fused_attention(
                q,
                k,
                v,
                causal=self.causal,
                window_size=self.window_size,
                dropout_p=self.dropout.p,
                training=self.training,
            )
            weights = None

        # ── Gated Attention (Qiu et al., NeurIPS 2025 Best Paper) ───────────────
        # Per-head sigmoid gate modulates the SDPA output before XSA/out_proj.
        # Gate input: first `attn_gate_window` dims of the post-norm block input
        # (hidden_states).  Sparse window keeps the gate head lightweight.
        # 2*sigmoid(z) is used so the zero-init projection gives gate=1 (identity)
        # at step 0; the head learns to suppress or amplify over training.
        gate_in = hidden_states[..., :self.attn_gate_window]  # (B, T, gw)
        gate = self.attn_gate_proj(gate_in)                   # (B, T, num_heads)

        # Fused gated-attention + XSA + position-0 guard + head merge (one kernel each way). The probe uses
        # non-zero gate logits / alpha so the zero-init (identity) state doesn't hide a wrong XSA path.
        _merged = None
        if (not output_attentions and context.is_cuda and context.shape[-1] & (context.shape[-1] - 1) == 0
                and _kt_on(context, "attn_post")):
            def _attn_probe(c_=context, v_=v, g_=gate):
                _pg = (0.5 * torch.cos(torch.arange(g_[:1].numel(), device=g_.device, dtype=torch.float32))
                       ).reshape(g_[:1].shape).to(g_.dtype).requires_grad_()
                _pa = torch.linspace(-0.8, 0.8, self.num_heads, device=g_.device,
                                     dtype=self.xsa_alpha.dtype).requires_grad_()
                return (c_[:1], v_[:1], _pg, _pa)

            _merged = _triton_attempt(
                "attn_post", (context.dtype, self.num_heads, self.num_kv_heads, self.head_dim, cache_pos > 0),
                lambda c_, v_, g_, a_: _moe_kernel.attn_post(c_, v_, g_, a_, cache_pos),
                lambda c_, v_, g_, a_: _moe_kernel.attn_post_eager(c_, v_, g_, a_, cache_pos),
                (context, v, gate, self.xsa_alpha),
                _attn_probe,
            )
        if _merged is not None:
            context = None
            merged_context = _merged
        else:
            gate = 2.0 * torch.sigmoid(gate)                      # (0, 2), =1.0 at init
            gate = gate.transpose(1, 2).unsqueeze(-1)             # (B, H, T, 1)
            context = context * gate
        # ────────────────────────────────────────────────────────────────────────

        if context is not None:
            v_xsa = v
            if self._xsa_v_reps > 1:
                # GQA/MQA-aware broadcast: mathematically identical to
                # v.repeat_interleave(self._xsa_v_reps, dim=1) (verified via
                # torch.equal), but avoids materializing a full copy of v.
                Bv, Hkv, Tv, Dv = v.shape
                v_xsa = (
                    v.reshape(Bv, Hkv, 1, Tv, Dv)
                    .expand(Bv, Hkv, self._xsa_v_reps, Tv, Dv)
                    .reshape(Bv, Hkv * self._xsa_v_reps, Tv, Dv)
                )

            v_norm = v_xsa.norm(dim=-1, keepdim=True).clamp(min=1e-6)
            v_unit = v_xsa / v_norm
            proj_coeff = (context * v_unit).sum(dim=-1, keepdim=True)
            # Learnable per-head gate (modded-nanoGPT style): tanh(alpha) in (-1, 1).
            # alpha starts at 0 -> gate=0 -> forward pass is numerically identical
            # to vanilla SA at step 0. Each head learns how much (if any) self-value
            # component to remove, rather than a hard, deterministic subtraction.
            gate = torch.tanh(self.xsa_alpha).view(1, -1, 1, 1)
            correction = gate * proj_coeff * v_unit

            # Position-0 guard: causal attention at position 0 has exactly one
            # reachable key (itself), so context[0] == v[0] identically, at every
            # step of training. XSA's projection therefore removes 100% of the
            # context for that head, for any gate > 0. This is structural, not
            # a training artifact -- there is no "context minus self" at
            # position 0 because there is no context other than self there.
            # Mask the correction off so the first token always keeps a
            # full-strength value contribution.
            T = context.shape[2]
            # Slice the pre-built buffer; no allocation per forward. Applied for T == 1 as well:
            # a 1-token input IS position 0 and must be masked exactly as it is in training.
            mask = self._xsa_pos_mask[cache_pos:cache_pos + T].to(dtype=context.dtype).view(1, 1, T, 1)
            correction = correction * mask

            context = context - correction

            merged_context = merge_heads(context)
        # ─────────────────────────────────────────────────────────────────────

        if skip_out_proj and self.tp_size == 1 and not output_attentions and not return_kv_cache_estimate:
            return merged_context

        output = self.out_proj(merged_context)

        
        if self.tp_size > 1:
            if self.tp_group is None:
                raise RuntimeError(
                    "DenseMHA was built with tp_size>1 but tp_group was never "
                    "set -- call GPT.set_tp_group(group) after construction, "
                    "before the first forward pass."
                )
            torch.distributed.all_reduce(output, op=torch.distributed.ReduceOp.SUM, group=self.tp_group)

        if output_attentions or return_kv_cache_estimate:
            cache_bytes = None
            if return_kv_cache_estimate:
                cache_bytes = estimate_kv_cache_bytes(
                    batch_size, seq_len, self.num_kv_heads, self.head_dim,
                    hidden_states.dtype, window_size=self.window_size,
                )
            return AttentionOutput(output=output, attention_weights=weights if output_attentions else None, kv_cache_bytes=cache_bytes)
        return output



class SwiGLU_FFN(nn.Module):
    """
    SwiGLU Feed Forward Network.
    
    
    """

    def __init__(self, d_model, ffn_mult=8/3, dropout=0.0, tp_size: int = 1):
        super().__init__()
        
        raw_hidden = d_model * ffn_mult
        hidden_dim = int(((raw_hidden + 63) // 64) * 64)
        if hidden_dim % tp_size != 0:
            raise ValueError(
                f"tensor_parallel_size ({tp_size}) must divide the FFN hidden_dim "
                f"({hidden_dim} = round({d_model} * {ffn_mult}) -> next mult of 64) "
                f"-- adjust ffn_mult or tensor_parallel_size."
            )
        local_hidden_dim = hidden_dim // tp_size
        self.tp_size = tp_size
        self.tp_group = None  # set post-construction via GPT.set_tp_group()
        self.W_gate = nn.Linear(d_model, local_hidden_dim, bias=False)
        self.W_value = nn.Linear(d_model, local_hidden_dim, bias=False)
        self.W_out = nn.Linear(local_hidden_dim, d_model, bias=False)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        if (self.tp_size == 1 and (not self.training or self.dropout.p == 0.0)
                and type(self.W_gate) is nn.Linear and type(self.W_value) is nn.Linear
                and type(self.W_out) is nn.Linear and x.is_cuda and _kt_on(x, "ffn")):
            out = _triton_attempt(
                "ffn", (x.dtype, x.ndim),
                lambda x_: _moe_kernel.swiglu_ffn(x_, self.W_gate.weight, self.W_value.weight, self.W_out.weight),
                lambda x_: self.W_out(F.silu(self.W_gate(x_)) * self.W_value(x_)),
                (x,), lambda: (x[:1],),
            )
            if out is not None:
                return out
        if (self.tp_size == 1 and type(self.W_gate.weight) is nn.Parameter
                and type(self.W_value.weight) is nn.Parameter):
            # One GEMM for both projections: [W_gate; W_value] stacked on the output dim, then split.
            # Parameters and state_dict keys are unchanged; only the cat (2*H*D weights) is extra work.
            gate_raw, value = F.linear(
                x, torch.cat((self.W_gate.weight, self.W_value.weight), dim=0)
            ).chunk(2, dim=-1)
        else:
            gate_raw = self.W_gate(x)
            value = self.W_value(x)

        hidden = None
        if _HAS_TRITON and gate_raw.is_cuda:
            hidden = _triton_attempt(
                "swiglu", (gate_raw.dtype,), _tri_swiglu, _swiglu_eager,
                (gate_raw.contiguous(), value.contiguous()), (gate_raw[:1].contiguous(), value[:1].contiguous()),
            )
        if hidden is None:
            hidden = F.silu(gate_raw) * value

        hidden = self.dropout(hidden)
        out = self.W_out(hidden)
        if self.tp_size > 1:
            if self.tp_group is None:
                raise RuntimeError(
                    "SwiGLU_FFN was built with tp_size>1 but tp_group was never "
                    "set -- call GPT.set_tp_group(group) after construction, "
                    "before the first forward pass."
                )
            torch.distributed.all_reduce(out, op=torch.distributed.ReduceOp.SUM, group=self.tp_group)
        return out  # Return delta only
class AryaSparseMoE(nn.Module):
    """
    DeepSeek-style Sparse Mixture of Experts.
    Expects input to be ALREADY normalized (Pre-LN).
    Returns the combined FFN delta.
    """

    def __init__(self, d_model, n_experts=8, n_shared=1, top_k=2, ffn_mult=8/3, dropout=0.0,
                 use_capacity_routing=False, capacity_factor=1.25, fixed_capacity: int | None = None):
        super().__init__()
        self.n_experts = n_experts
        self.n_routed = n_experts - n_shared
        self.top_k = top_k
        self.d_model = d_model
        self.use_capacity_routing = use_capacity_routing
        self.capacity_factor = capacity_factor
        self.fixed_capacity = fixed_capacity

        if self.n_routed < top_k:
            raise ValueError(
                f"AryaSparseMoE config error: n_experts={n_experts} - n_shared={n_shared} "
                f"= {self.n_routed} routed experts, but top_k={top_k} routing needs at "
                f"least {top_k} routed experts to choose from."
            )

        self.shared_expert = SwiGLU_FFN(d_model, ffn_mult, dropout=dropout)
        self.routed_experts = nn.ModuleList([
            SwiGLU_FFN(d_model, ffn_mult, dropout=dropout)
            for _ in range(self.n_routed)
        ])

        self.router = nn.Linear(d_model, self.n_routed, bias=False)
        self.expert_bias = nn.Parameter(torch.zeros(self.n_routed), requires_grad=False)
        self.aux_loss_weight = float(getattr(alpha_config, 'moe_aux_loss_weight', 0.01))
        self.last_aux_loss = torch.zeros((), dtype=torch.float32)

    def forward(self, x):
        B, T, D = x.shape

        
        shared_out = self.shared_expert(x)

        
        router_logits = self.router(x)
        _kt_router = None
        if x.is_cuda and _kt_on(router_logits, "router"):
            _lg = router_logits.reshape(B * T, self.n_routed)
            if self.training and torch.is_grad_enabled():
                _kt_router = _triton_attempt(
                    "router_train", (_lg.dtype, self.n_routed, self.top_k),
                    lambda l_: _moe_kernel.router_topk(l_, self.top_k, self.expert_bias, True),
                    lambda l_: _moe_kernel.router_eager(l_, self.expert_bias, self.top_k, True),
                    (_lg,), lambda: (_lg[:512],),
                )
            else:
                _kt_router = _triton_attempt(
                    "router_eval", (_lg.dtype, self.n_routed, self.top_k),
                    lambda l_: _moe_kernel.router_topk(l_, self.top_k),
                    lambda l_: _moe_kernel.router_eager(l_, None, self.top_k, False),
                    (_lg,), lambda: (_lg[:512],),
                )
        if _kt_router is not None:
            if len(_kt_router) == 4:   # training: probs, idx, mean softmax prob P, per-expert assignment counts
                topk_probs, topk_indices, _P, _cnt = _kt_router
                self.last_aux_loss = self.aux_loss_weight * self.n_routed * ((_cnt.float() / (B * T)) * _P).sum()
            else:
                topk_probs, topk_indices = _kt_router
                self.last_aux_loss = torch.zeros((), device=x.device, dtype=torch.float32)
        elif self.training and torch.is_grad_enabled():
            biased_logits = router_logits.float() + self.expert_bias.float()
            _, topk_indices = biased_logits.topk(self.top_k, dim=-1)
            full_probs = F.softmax(router_logits.float(), dim=-1)
            topk_probs = full_probs.gather(-1, topk_indices).to(router_logits.dtype)
            topk_probs = topk_probs / topk_probs.sum(dim=-1, keepdim=True).clamp_min(1e-9)
            one_hot = F.one_hot(topk_indices, num_classes=self.n_routed).float()
            f = one_hot.mean(dim=(0, 1))
            P = full_probs.mean(dim=(0, 1))
            self.last_aux_loss = self.aux_loss_weight * self.n_routed * (f * P).sum()
            # Do not mutate expert_bias here. Checkpoint recomputation runs
            # this forward a second time, so an in-forward update would apply
            # twice per optimizer step and make routing state nondeterministic.
        else:
            full_probs = F.softmax(router_logits.float(), dim=-1)
            topk_probs, topk_indices = full_probs.topk(self.top_k, dim=-1)
            topk_probs = (topk_probs / topk_probs.sum(dim=-1, keepdim=True).clamp_min(1e-9)).to(router_logits.dtype)
            self.last_aux_loss = torch.zeros((), device=x.device, dtype=torch.float32)
        gate_weights = topk_probs

        x_flat = x.view(B * T, D)
        gate_flat = gate_weights.view(B * T, self.top_k)
        idx_flat = topk_indices.view(B * T, self.top_k)

        if self.use_capacity_routing:
            routed_out = self._forward_capacity_routed(x_flat, gate_flat, idx_flat)
            routed_out = routed_out.view(B, T, D)
        elif _moe_kernel_ready():
            # shared + routed is fused into the combine kernel; verified vs the eager loop on first use.
            _shared_flat = shared_out.reshape(B * T, D)
            out = _triton_attempt(
                "moe_experts", (x_flat.dtype, self.n_routed, self.top_k),
                lambda x_, g_, i_, s_: _moe_kernel.fused_moe_experts(self.routed_experts, x_, g_, i_, shared=s_),
                lambda x_, g_, i_, s_: self._eager_loop(x_, g_, i_) + s_,
                (x_flat, gate_flat, idx_flat, _shared_flat),
                lambda: (x_flat[:256], gate_flat[:256], idx_flat[:256], _shared_flat[:256]),
            ) if x_flat.is_cuda else None
            if out is not None:
                return out.view(B, T, D), self.last_aux_loss
            routed_out = self._eager_loop(x_flat, gate_flat, idx_flat).view(B, T, D)
        else:
            routed_out = self._eager_loop(x_flat, gate_flat, idx_flat).view(B, T, D)

        out = shared_out + routed_out
        return out, self.last_aux_loss

    def _eager_loop(self, x_flat, gate_flat, idx_flat):
        """Per-expert Python loop -- the reference implementation and fallback."""
        routed_out = torch.zeros_like(x_flat)
        expert_masks = F.one_hot(idx_flat, num_classes=self.n_routed).bool().any(dim=1)
        for expert_id in range(self.n_routed):
            expert = self.routed_experts[expert_id]
            token_mask = expert_masks[:, expert_id]
            selected_tokens = x_flat[token_mask]
            if selected_tokens.shape[0] == 0:
                continue
            expert_out = expert(selected_tokens)
            weight = (
                (idx_flat[token_mask] == expert_id).float() * gate_flat[token_mask]
            ).sum(dim=-1)
            routed_out[token_mask] += weight.unsqueeze(-1) * expert_out
        return routed_out

    def _forward_capacity_routed(self, x_flat, gate_flat, idx_flat):
        n_tokens, D = x_flat.shape
        n_routed, capacity = self.n_routed, self._capacity(n_tokens)

        flat_expert_ids = idx_flat.reshape(-1)
        flat_gates = gate_flat.reshape(-1)
        flat_token_ids = torch.arange(n_tokens, device=x_flat.device).unsqueeze(1).expand(-1, self.top_k).reshape(-1)

        
        one_hot = F.one_hot(flat_expert_ids, num_classes=n_routed).to(torch.float32)
        position_in_expert = (one_hot.cumsum(dim=0) * one_hot).sum(dim=1).long() - 1
        keep = position_in_expert < capacity
        safe_position = position_in_expert.clamp(min=0, max=capacity - 1)

        flat_slot = flat_expert_ids * capacity + safe_position

        gathered_x = x_flat[flat_token_ids] * keep.unsqueeze(-1).to(x_flat.dtype)
        dispatch = torch.zeros(n_routed * capacity, D, device=x_flat.device, dtype=x_flat.dtype)
        dispatch.index_add_(0, flat_slot, gathered_x)
        dispatch = dispatch.view(n_routed, capacity, D)

        expert_out = torch.stack(
            [expert(dispatch[e]) for e, expert in enumerate(self.routed_experts)],
            dim=0,
        ).view(n_routed * capacity, D)

        contrib = expert_out[flat_slot] * keep.unsqueeze(-1).to(x_flat.dtype) * flat_gates.unsqueeze(-1)
        routed_out = torch.zeros(n_tokens, D, device=x_flat.device, dtype=x_flat.dtype)
        routed_out.index_add_(0, flat_token_ids, contrib)
        return routed_out

    def _capacity(self, n_tokens) -> int:
        if self.fixed_capacity is not None:
            return self.fixed_capacity
        total_assignments = n_tokens * self.top_k
        avg_load = -(-total_assignments // self.n_routed)
        numer = int(round(self.capacity_factor * 1000))
        capacity = -(-(avg_load * numer) // 1000)
        return capacity + 1


def build_attention_layers(
    num_layers: int,
    hidden_size: int,
    num_heads: int,
    dropout: float = 0.0,
    bias: bool = True,
    num_kv_heads: int | None = None,
    pattern: str = "dense",
    max_seq_len: int = 4096,
    sliding_window_size: int = 256,
    tp_size: int = 1,
    rope_dim: int | None = None,
):
    
    if num_kv_heads is None:
        num_kv_heads = num_heads

    layers = []
    layer_types = []

    if pattern == "dense":
        for _ in range(num_layers):
            layers.append(DenseMHA(
                hidden_size=hidden_size, num_heads=num_heads, num_kv_heads=num_kv_heads,
                dropout=dropout, bias=bias, max_seq_len=max_seq_len,
                window_size=None, tp_size=tp_size, rope_dim=rope_dim,
            ))
            layer_types.append("dense")

    elif pattern == "pyramid_swa":
        
        if num_kv_heads <= 1:
            pyramid_kv_heads = [1, 1, 1]
        else:
            import math as _math
            
            mqa_floor = max(2, _math.ceil(num_kv_heads / 4))
            mqa_floor = min(mqa_floor, num_kv_heads)  # never exceed full heads
            pyramid_kv_heads = sorted(set([
                mqa_floor,
                max(mqa_floor, _math.ceil(num_kv_heads / 2)),
                num_kv_heads,
            ]))
            
            while len(pyramid_kv_heads) < 3:
                pyramid_kv_heads.append(num_kv_heads)
            pyramid_kv_heads = pyramid_kv_heads[:3]

        for i in range(num_layers):
            pos_in_group = i % 4
            is_last_layer = (i == num_layers - 1)
            
            if pos_in_group == 3 or is_last_layer:
                layers.append(DenseMHA(
                    hidden_size=hidden_size, num_heads=num_heads, num_kv_heads=num_kv_heads,
                    dropout=dropout, bias=bias, max_seq_len=max_seq_len,
                    window_size=None, tp_size=tp_size, rope_dim=rope_dim,
                ))
                layer_types.append("global")
            else:
                kv_heads_here = pyramid_kv_heads[pos_in_group]
                layers.append(DenseMHA(
                    hidden_size=hidden_size, num_heads=num_heads, num_kv_heads=kv_heads_here,
                    dropout=dropout, bias=bias, max_seq_len=max_seq_len,
                    window_size=sliding_window_size, tp_size=tp_size, rope_dim=rope_dim,
                ))
                layer_types.append("swa")
    elif pattern == "mod":
        
        for i in range(num_layers):
            layers.append(DenseMHA(
                hidden_size=hidden_size, num_heads=num_heads, num_kv_heads=num_kv_heads,
                dropout=dropout, bias=bias, max_seq_len=max_seq_len,
                window_size=None, tp_size=tp_size, rope_dim=rope_dim,
            ))
            is_full_dense = (i % 4 == 3) or (i == num_layers - 1)
            layer_types.append("mod_dense" if is_full_dense else "mod")
    else:
        raise ValueError(
            f"Unknown attention pattern {pattern!r} -- supported patterns: "
            f"'dense', 'pyramid_swa', 'mod'. "
            f"(BigBird/conv-hybrid/LSA/HCA patterns have been removed.)"
        )

    return nn.ModuleList(layers), layer_types


class AryaBlock(nn.Module):
    """Single transformer block: Layer Norm → Attention → MLP (with residual connections).

    `attn_module` is DenseMHA,
    pre-built by GPT via build_attention_layers. Every layer is fully
    standalone -- no cross-layer attention/KV sharing of any kind.
    `role` is purely a label ("dense" | "lsa") for inspection/debugging.
    """

    def __init__(self, config, attn_module: nn.Module, role: str = "dense"):
        super().__init__()
        self.ln_1 = RMSNorm(config.n_embd, eps=1e-5)
        # Pre-LN for the MLP branch: ensure router and experts see the
        # identical normalized tensor the MLPs expect.
        self.ln_2 = RMSNorm(config.n_embd, eps=1e-5)
        self.attn = attn_module
        self.role = role  # "dense" | "lsa"

       
        self.use_moe = getattr(config, 'use_moe', False)
        if self.use_moe:
            if getattr(config, 'tp_size', 1) > 1:
                raise ValueError(
                    "tensor_parallel_size > 1 is not supported together with use_moe=True -- "
                    "MoE needs its own (unimplemented here) expert-parallel sharding scheme, "
                    "column/row-parallel head splitting does not apply to expert routing. "
                    "Set use_moe=False or tensor_parallel_size=1."
                )
            self.ffn = AryaSparseMoE(
                d_model=config.n_embd,
                n_experts=config.n_experts,
                n_shared=config.n_shared,
                top_k=getattr(config, 'moe_top_k', 2),
                ffn_mult=config.ffn_mult,
                dropout=config.dropout,
                use_capacity_routing=False,
                fixed_capacity=getattr(config, 'moe_fixed_capacity', None),
            )
        else:
            self.ffn = SwiGLU_FFN(config.n_embd, config.ffn_mult, dropout=config.dropout, tp_size=getattr(config, 'tp_size', 1))

    def forward(self, x, kv_cache=None, cache_pos=0):
        """Every attention layer (DenseMHA, LSA) is standalone and
        computes its own K/V from `x` alone --
        no external_kv, no cross-layer sharing, no produced_kv to pass on.
        """
        normed = self.ln_1(x)
        if kv_cache is None:
            attn_delta = module_output_tensor(self.attn(normed))
        else:
            attn_delta = module_output_tensor(self.attn(normed, kv_cache=kv_cache, cache_pos=cache_pos))
        _fused = None
        if x.is_cuda and _kt_on(x, "add_rmsnorm"):
            _w, _eps = self.ln_2.weight, self.ln_2.eps
            _odt = _kt_norm_dtype(x, _w, _eps)
            _fused = _triton_attempt(
                "add_rmsnorm", (x.dtype, attn_delta.dtype, _w.dtype),
                lambda x_, d_, w_: _moe_kernel.add_rmsnorm(x_, d_, w_, _eps, _odt),
                lambda x_, d_, w_: ((x_ + d_), _rmsnorm_eager(x_ + d_, w_, _eps)),
                (x, attn_delta, _w),
                lambda: (x[:1], attn_delta[:1],
                         torch.linspace(0.5, 1.5, _w.numel(), device=_w.device, dtype=_w.dtype).requires_grad_()),
            )
        if _fused is not None:
            x, ffn_in = _fused
        else:
            x = x + attn_delta
            ffn_in = self.ln_2(x)
        ffn_result = self.ffn(ffn_in)
        if self.use_moe:
            ffn_delta, moe_aux = ffn_result
            return x + ffn_delta, moe_aux
        return x + ffn_result


class MoDBlock(nn.Module):

    def __init__(self, config, inner: nn.Module, capacity: float = 0.125):
        super().__init__()
        self.inner = inner
        self.capacity = capacity
        self.keep_target = float(capacity)
        self._budget_coeff = float(getattr(config, 'mod_budget_coeff', 0.05))
        self._entropy_coeff = float(getattr(config, 'mod_gate_entropy_coeff', 0.0))
        self.hard_eval = bool(getattr(config, 'mod_hard_eval', False))

        self._router_norm = RMSNorm(config.n_embd, eps=1e-5)
        self.router = nn.Linear(config.n_embd, 1, bias=True)
        self.last_gate_aux = None

    def forward(self, x, kv_cache=None, cache_pos=0):
        gate = torch.sigmoid(self.router(self._router_norm(x)))

        aux = torch.zeros((), device=x.device, dtype=torch.float32)
        if self.training:
            g = gate.float()
            if self._budget_coeff > 0.0:
                aux = aux + self._budget_coeff * (g.mean() - self.keep_target) ** 2
            if self._entropy_coeff > 0.0:
                gc = g.clamp(1e-6, 1.0 - 1e-6)
                aux = aux + self._entropy_coeff * (
                    -(gc * gc.log() + (1 - gc) * (1 - gc).log()).mean()
                )
            # BUGFIX: `aux` must stay attached to the graph -- returning
            # aux.detach() made the MoD budget/entropy regularizer contribute
            # exactly zero gradient to the router, i.e. it was inert.
            self.last_gate_aux = aux.detach()   # detached copy, for logging only
        else:
            aux = torch.zeros((), device=x.device, dtype=torch.float32)
            self.last_gate_aux = aux

        kw = {} if kv_cache is None else {"kv_cache": kv_cache, "cache_pos": cache_pos}
        if self.inner.use_moe:
            inner_out, moe_aux = self.inner(x, **kw)
        else:
            inner_out = self.inner(x, **kw)
            moe_aux = torch.zeros((), device=x.device, dtype=torch.float32)
        if gate.dtype == x.dtype == inner_out.dtype:
            out = torch.lerp(x, inner_out, gate)   # == x + gate * (inner_out - x), single fused kernel
        else:
            out = x + gate * (inner_out - x)
        return (out, moe_aux), aux

    def extra_repr(self) -> str:
        return (
            f"gate_based=True,keep_target={self.keep_target},"
            f"budget_coeff={self._budget_coeff},entropy_coeff={self._entropy_coeff}"
        )


class MTPHead(nn.Module):
    """Residual shared-head auxiliary for a future-token prediction depth."""

    def __init__(self, hidden_size: int, ln_f_weight: torch.Tensor):
        super().__init__()
        self.weight = nn.Parameter(ln_f_weight.detach().clone())
        self.proj = nn.Linear(hidden_size, hidden_size, bias=False)
        nn.init.zeros_(self.proj.weight)
        self.eps = 1e-5

    def forward(self, hidden: torch.Tensor) -> torch.Tensor:
        # F.rms_norm handles the dtype internally -- no manual fp32 upcast
        # allocation, no (B, T, D) intermediate tensor, free for any dtype.
        h_norm = _native_rms_norm(hidden, (self.weight.shape[0],), self.weight.to(hidden.dtype), self.eps)
        return h_norm + self.proj(h_norm)


class MultiTokenPrediction(nn.Module):

    def __init__(self, config, depth: int = 1, ln_f_weight: torch.Tensor | None = None):
        super().__init__()
        self.depth = depth
        initial_weight = (
            ln_f_weight if ln_f_weight is not None else torch.ones(config.n_embd)
        )
        self.heads = nn.ModuleList([
            MTPHead(config.n_embd, initial_weight) for _ in range(depth)
        ])
        self.last_raw = None   # unweighted mean MTP loss from the last compute_loss (for logging)

    def compute_loss(
        self,
        x: torch.Tensor,               # (B, T, D) -- pre-lm_head hidden states
        targets: torch.Tensor,         # (B, T) -- already the +1 shifted targets
        lm_head_weight: torch.Tensor,  # (vocab_size, D) -- tied weight
        mtp_lambda: float = 0.3,
    ) -> torch.Tensor:
        
        
        total = torch.zeros((), device=x.device, dtype=torch.float32)
        valid_depths = 0
        for k, head in enumerate(self.heads):
            offset = k + 1   # additional token offset beyond the main +1 shift
            Tk = x.size(1) - offset
            if Tk <= 0:
                continue
            h = head(x[:, :Tk])                      # (B, Tk, D)
            tgt_k = targets[:, offset:]               # (B, Tk)
            # Keep sum reduction: all-ignored chunks must contribute zero, not NaN.
            loss_k = chunked_cross_entropy(h, lm_head_weight, tgt_k, ignore_index=-1)
            total = total + loss_k
            valid_depths += 1
        if valid_depths == 0:
            self.last_raw = torch.zeros((), device=x.device, dtype=torch.float32)
            return self.last_raw
        self.last_raw = (total / valid_depths).detach()
        return mtp_lambda * (total / valid_depths)


@dataclass
class GPTConfig:
    """Configuration for GPT model."""
    block_size: int = _alpha_config_attr('CONTEXT', 1024)
    vocab_size: int = _alpha_config_attr('vocab_size', 50304)
    n_layer: int = _alpha_config_attr('numberoflayers', 12)
    n_head: int = _alpha_config_attr('numberofheads', 12)
    n_embd: int = _alpha_config_attr('D_MODEL', 768)
    dropout: float = _alpha_config_attr('dropout', 0.0)

    bias: bool = _alpha_config_attr('bias', False)
    ffn_mult: float = _alpha_config_attr('ffn_mult', 4.0)
    
    d_rope: int | None = _alpha_config_attr('d_rope', None)
    
    top_k: int = _alpha_config_attr('top_k', 64)
    n_experts: int = _alpha_config_attr('n_experts', 8)
    n_shared: int = _alpha_config_attr('n_shared', 1)
    use_moe: bool = _alpha_config_attr('use_moe', False)
    
    use_liger: bool = _alpha_config_attr('use_liger', False)
    
    moe_top_k: int = _alpha_config_attr('moe_top_k', 2)

    gradient_checkpointing: bool = _alpha_config_attr('gradient_checkpointing', False)          # trade ~20-30% more compute for substantially less activation memory (grows with n_layer -- see GPT.forward)
    num_kv_heads: int | None = _alpha_config_attr('num_kv_heads', None)
    
    pattern: str = _alpha_config_attr('pattern', 'pyramid_swa')
    
    sliding_window_size: int = _alpha_config_attr('sliding_window_size', 256)
    
    tp_size: int = _alpha_config_attr('tensor_parallel_size', 1)

    
    gen_headroom: int = _alpha_config_attr('gen_headroom', 0)
    
    max_gen_tokens: int | None = _alpha_config_attr('max_gen_tokens', None)

   
    mod_capacity: float = _alpha_config_attr('mod_capacity', 0.125)

    
    use_mtp: bool = _alpha_config_attr('use_mtp', False)
    mtp_depth: int = _alpha_config_attr('mtp_depth', 1)
    mtp_lambda: float = _alpha_config_attr('mtp_lambda', 0.3)

   
    use_nvfp4: bool = _alpha_config_attr('use_nvfp4', False)

    
    use_int8: bool = _alpha_config_attr('use_int8', False)
    int8_group_size: int = _alpha_config_attr('int8_group_size', 128)

    mod_gate_entropy_coeff: float = _alpha_config_attr('mod_gate_entropy_coeff', 0.01)

    # Engram static n-gram memory (arXiv:2601.07372). Tuples, not lists:
    # dataclass rejects mutable defaults.
    use_engram: bool = _alpha_config_attr('use_engram', False)
    engram_layer_ids: tuple = tuple(_alpha_config_attr('engram_layer_ids', (1,)))
    engram_max_ngram: int = _alpha_config_attr('engram_max_ngram', 3)
    engram_vocab_size: tuple = tuple(_alpha_config_attr('engram_vocab_size', (8192, 8192)))
    engram_embed_per_ngram: int = _alpha_config_attr('engram_embed_per_ngram', 128)
    engram_n_heads: int = _alpha_config_attr('engram_n_heads', 4)
    engram_kernel_size: int = _alpha_config_attr('engram_kernel_size', 4)
    engram_seed: int = _alpha_config_attr('engram_seed', 0)
    # Set automatically from the tokenizer lookup at build time; saved in
    # checkpoints so GPT(config) can be rebuilt without a tokenizer.
    engram_compressed_vocab: int = _alpha_config_attr('engram_compressed_vocab', 0)

class GPT(nn.Module):
    """Full GPT language model"""

    def __init__(self, config, engram_lookup=None):
        super().__init__()
        if config.vocab_size is None:
            raise ValueError("config.vocab_size must be set")
        if config.block_size is None:
            raise ValueError("config.block_size must be set")
        self.config = config

        
        
        head_dim = config.n_embd // config.n_head
        rope_dim = head_dim if getattr(config, 'd_rope', None) is None else config.d_rope
        
        gen_headroom = max(0, int(getattr(config, 'gen_headroom', 0)))
        rope_table_len = config.block_size + gen_headroom

        attn_layers, self.layer_types = build_attention_layers(
            num_layers=config.n_layer,
            hidden_size=config.n_embd,
            num_heads=config.n_head,
            dropout=config.dropout,
            bias=config.bias,
            num_kv_heads=config.num_kv_heads,
            pattern=config.pattern,
            max_seq_len=rope_table_len,
            sliding_window_size=config.sliding_window_size,
            tp_size=config.tp_size,
            rope_dim=rope_dim,
        )

        
        mod_capacity = float(getattr(config, 'mod_capacity', 0.125))
        blocks = []
        for i in range(config.n_layer):
            block = AryaBlock(config, attn_layers[i], role=self.layer_types[i])
            if self.layer_types[i] == "mod":
                
                block = MoDBlock(config, block, capacity=mod_capacity)
            blocks.append(block)

        # Hoist once: avoid an isinstance(block, MoDBlock) check (and the
        # use_moe lookup that decides whether a block yields a moe_aux
        # tensor) on every block on every forward.
        self._no_kw = [{} for _ in blocks]   # shared empty kwargs: no per-layer dict alloc
        self._block_flags = [
            (isinstance(b, MoDBlock), b.inner.use_moe if isinstance(b, MoDBlock) else b.use_moe)
            for b in blocks
        ]

        self.transformer = nn.ModuleDict(dict(
            wte=nn.Embedding(config.vocab_size, config.n_embd),
            drop=nn.Dropout(config.dropout),
            h=nn.ModuleList(blocks),
            ln_f=RMSNorm(config.n_embd, eps=1e-5),
        ))
        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)

        # ── Engram (optional) ────────────────────────────────────────────
        self.engram_hasher = None
        self.engram_layers = nn.ModuleDict()
        self._engram_layer_ids = frozenset()
        if bool(getattr(config, 'use_engram', False)):
            if Engram is None:
                raise ImportError("use_engram=True but engram.py could not be imported")
            ids = tuple(int(l) for l in config.engram_layer_ids)
            if not ids or any(not (0 <= l < config.n_layer) for l in ids) or len(set(ids)) != len(ids):
                raise ValueError(
                    f"engram_layer_ids={ids} must be unique and within [0, {config.n_layer})")
            if engram_lookup is not None:
                config.engram_compressed_vocab = int(engram_lookup.max()) + 1
            if int(getattr(config, 'engram_compressed_vocab', 0)) <= 0:
                raise ValueError(
                    "use_engram=True needs engram_lookup (built from the tokenizer via "
                    "engram.CompressedTokenizer) or a config with engram_compressed_vocab set")
            self.engram_hasher = NgramHasher(
                layer_ids=ids,
                max_ngram=config.engram_max_ngram,
                vocab_size_per_ngram=config.engram_vocab_size,
                n_heads=config.engram_n_heads,
                compressed_vocab=config.engram_compressed_vocab,
                raw_vocab_size=config.vocab_size,
                seed=config.engram_seed,
                lookup=engram_lookup,
            )
            self.engram_layers = nn.ModuleDict({
                str(l): Engram(
                    hidden_size=config.n_embd,
                    head_sizes=self.engram_hasher.head_sizes[i],
                    max_ngram=config.engram_max_ngram,
                    embed_per_ngram=config.engram_embed_per_ngram,
                    n_heads=config.engram_n_heads,
                    kernel_size=config.engram_kernel_size,
                )
                for i, l in enumerate(ids)
            })
            self._engram_layer_ids = frozenset(ids)

        
        use_mtp = bool(getattr(config, 'use_mtp', False))
        mtp_depth = int(getattr(config, 'mtp_depth', 1))
        self.mtp = MultiTokenPrediction(config, depth=mtp_depth) if use_mtp else None
       
        self._mtp_lambda = float(getattr(config, 'mtp_lambda', 0.3))

        
        self.apply(self._init_weights)
        for _eg in self.engram_layers.values():
            _eg.reset_parameters()   # tables std=0.01 (apply() above overwrote it with 0.02)

        
        # apply(_init_weights) above re-inits every nn.Linear to N(0, 0.02),
        # including the always-on attn_gate_proj. Restore its zero-init so the
        # gate is identity (2*sigmoid(0) = 1) at step 0.
        for block in self.transformer.h:
            real_block = block.inner if isinstance(block, MoDBlock) else block
            if isinstance(real_block.attn, DenseMHA):
                nn.init.zeros_(real_block.attn.attn_gate_proj.weight)

        for block in self.transformer.h:
            if isinstance(block, MoDBlock):
                with torch.no_grad():
                    block.router.bias.fill_(2.0)

        if self.mtp is not None:
            with torch.no_grad():
                for head in self.mtp.heads:
                    head.weight.copy_(self.transformer.ln_f.weight)
                    head.proj.weight.zero_()

        
        self.transformer.wte.weight = self.lm_head.weight

        
        residual_std = 0.02 / math.sqrt(config.n_layer)
        for name, p in self.named_parameters():
            if name.endswith("out_proj.weight") or name.endswith("W_out.weight"):
                torch.nn.init.normal_(p, mean=0.0, std=residual_std)

        
        self._use_nvfp4: bool = False
        self._use_int8: bool = False
        self._quant_mode = None

        if _nvfp4_enabled():
            n_quantized = apply_nvfp4_to_model(self)
            self._use_nvfp4 = True
            self._quant_mode = "nvfp4"
            warnings.warn(
                f"[nvFP4/RHT] torchao quantized {n_quantized} Linear layers "
                f"(Blackwell SM100+). Weight: FP4 e2m1 per-block-16 RHT. "
                f"Activation: FP4 dynamic per-tensor scale (FP8 intermediate). "
                f"Fused GEMM kernel emitted by torch.compile. "
                f"Skipped: wte, lm_head, norms, router, gate, small/misaligned linears. "
                f"Verify val_loss vs bf16 baseline before committing to a long run.",
                stacklevel=2,
            )
        elif _int8_enabled():
            gs = int(getattr(config, 'int8_group_size', 128))
            n_quantized = apply_int8_to_model(self, group_size=gs)
            self._use_int8 = True
            self._quant_mode = "int8"
            warnings.warn(
                f"[INT8/Jetfire] torchao quantized {n_quantized} Linear layers "
                f"(T4/Turing+). Weight: INT8 per-group-{gs} symmetric. "
                f"Activation: INT8 dynamic per-tensor scale (runtime, no calibration). "
                f"Fused kernel emitted by torch.compile via Inductor. "
                f"Skipped: wte, lm_head, norms, router, gate, small linears. "
                f"Verify val_loss vs fp16 baseline before a long run.",
                stacklevel=2,
            )

        
        
        self.transformer.wte.weight = self.lm_head.weight
        assert self.transformer.wte.weight is self.lm_head.weight


        
        _init_device = (
            torch.device("cuda", torch.cuda.current_device())
            if torch.cuda.is_available()
            else torch.device("cpu")
        )
        kernel.print_attention_backend(_init_device)
        
        kernel.warmup_attention_backend(_init_device)

    def set_tp_group(self, group) -> None:
        
        if self.config.tp_size <= 1:
            return
        for block in self.transformer.h:
            
            real_block = block.inner if isinstance(block, MoDBlock) else block
            real_block.attn.tp_group = group
            if hasattr(real_block, "ffn") and real_block.ffn is not None:
                real_block.ffn.tp_group = group

        
        if torch.distributed.is_initialized() and group is not None:
            for block in self.transformer.h:
                real_block = block.inner if isinstance(block, MoDBlock) else block
                attn = real_block.attn
                if not isinstance(attn, DenseMHA):
                    continue
                if attn.num_kv_heads >= attn.tp_size:
                    continue
                with torch.no_grad():
                    # Full qkv projection tensor is the canonical object to align.
                    torch.distributed.broadcast(attn.qkv_proj.weight.data, src=0, group=group)
                    if attn.qkv_proj.bias is not None:
                        torch.distributed.broadcast(attn.qkv_proj.bias.data, src=0, group=group)

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
            if module.bias is not None:
                torch.nn.init.zeros_(module.bias)
        elif isinstance(module, nn.Embedding):
            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)

    def get_num_params(self, non_embedding=True):
        """Count number of parameters"""
        n_params = sum(p.numel() for p in self.parameters())
        if non_embedding:
            n_params -= self.transformer.wte.weight.numel()
        return n_params

    def forward(self, idx, targets=None, need_logits=True, kv_cache=None):
        
        idx = idx.long()
        if targets is not None:
            targets = targets.long()
        b, t = idx.size()
        # Plain check, not torch._check with a message closure (that closure forces a graph break).
        if t > self.config.block_size:
            raise ValueError(f"Sequence too long: {t} > {self.config.block_size}")
        cache_pos = 0
        if kv_cache is not None:
            if not self.supports_kv_cache:
                raise NotImplementedError("this model config (e.g. Engram) does not support kv_cache")
            cache_pos = kv_cache.pos

        tok_emb = self.transformer.wte(idx)        # (B, T, n_embd)
        x = self.transformer.drop(tok_emb)

       
        mod_gate_loss = torch.zeros((), device=x.device, dtype=torch.float32)
        moe_aux_loss = torch.zeros((), device=x.device, dtype=torch.float32)

        engram_hashes = self.engram_hasher(idx) if self.engram_hasher is not None else None

        # Static-per-call decisions, resolved once instead of once per layer.
        engram_on = engram_hashes is not None
        use_ckpt = self.config.gradient_checkpointing and self.training
        kws = (self._no_kw if kv_cache is None else
               [{"kv_cache": kv_cache.layer(i), "cache_pos": cache_pos} for i in range(len(self._no_kw))])

        for layer_i, (block, (is_mod, has_moe), kw) in enumerate(zip(self.transformer.h, self._block_flags, kws)):
            if engram_on and layer_i in self._engram_layer_ids:
                # Official placement: h = h + Engram(h, ids) BEFORE the block's attention.
                x = x + self.engram_layers[str(layer_i)](x, engram_hashes[layer_i])
            if use_ckpt:
                if is_mod:
                    (x, moe_aux), gate_aux = torch.utils.checkpoint.checkpoint(
                        block, x, use_reentrant=False
                    )
                    mod_gate_loss = mod_gate_loss + gate_aux.float()
                elif has_moe:
                    x, moe_aux = torch.utils.checkpoint.checkpoint(
                        block, x, use_reentrant=False
                    )
                else:
                    x = torch.utils.checkpoint.checkpoint(
                        block, x, use_reentrant=False
                    )
                    moe_aux = None
            elif is_mod:
                (x, moe_aux), gate_aux = block(x, **kw)
                mod_gate_loss = mod_gate_loss + gate_aux.float()
            elif has_moe:
                x, moe_aux = block(x, **kw)
            else:
                x = block(x, **kw)
                moe_aux = None
            if moe_aux is not None:
                moe_aux_loss = moe_aux_loss + moe_aux.float()

        pre_ln_f = x
        x = self.transformer.ln_f(x)

        if targets is not None:
            mtp_loss = torch.zeros((), device=x.device, dtype=torch.float32)
            if self.mtp is not None:
                mtp_loss = self.mtp.compute_loss(
                    pre_ln_f, targets, self.lm_head.weight, mtp_lambda=self._mtp_lambda
                )
                self.last_mtp_loss = self.mtp.last_raw            # raw (unweighted), for logging
            else:
                self.last_mtp_loss = None

            liger_ce = None   # must exist even when the Liger path is not taken
            if (not need_logits
                    and _LIGER_CE_ON
                    and _liger_fused_ce_cuda_gate(x)):
                liger_ce = _liger_fused_ce_attempt(x, self.lm_head.weight, targets)
            if liger_ce is not None:
                total_loss = liger_ce + mtp_loss + mod_gate_loss + moe_aux_loss
                self.last_ce_loss = liger_ce.detach()
                self.last_moe_aux_loss = moe_aux_loss.detach()
                self.last_total_loss = total_loss.detach()
                return None, total_loss

            if not need_logits:
                
                ce_loss = chunked_cross_entropy(x, self.lm_head.weight, targets, ignore_index=-1)
                logits = None
            else:
                logits = F.linear(x, self.lm_head.weight)
                ce_loss = F.cross_entropy(
                    logits.reshape(-1, logits.size(-1)).float(),
                    targets.reshape(-1),
                    ignore_index=-1,
                )

            total_loss = ce_loss + mtp_loss + mod_gate_loss + moe_aux_loss
            self.last_ce_loss = ce_loss.detach()
            self.last_moe_aux_loss = moe_aux_loss.detach()
            self.last_total_loss = total_loss.detach()
        else:
            logits = F.linear(x[:, [-1], :], self.lm_head.weight)  # (B, 1, vocab_size)
            ce_loss = None
            total_loss = None

        if kv_cache is not None:
            kv_cache.pos += t
        return logits, total_loss

    @property
    def supports_kv_cache(self) -> bool:
        return (
            self.engram_hasher is None
            and all(
                isinstance((b.inner if isinstance(b, MoDBlock) else b).attn, DenseMHA)
                and (b.inner if isinstance(b, MoDBlock) else b).attn.causal
                for b in self.transformer.h
            )
        )

    def new_kv_cache(self, max_len: int | None = None) -> "KVCache":
        """Cache sized for `max_len` tokens (default and upper bound: block_size)."""
        cap = self.config.block_size if max_len is None else min(int(max_len), self.config.block_size)
        return KVCache(len(self.transformer.h), cap)

    @torch.no_grad()
    def generate(self, idx, max_new_tokens=None, temperature=1.0, top_k=None, use_cache=True):
        
        if max_new_tokens is None:
            max_new_tokens = getattr(self.config, 'max_gen_tokens', None)
            if max_new_tokens is None:
                raise ValueError(
                    "generate() needs max_new_tokens, either passed "
                    "directly or set as max_gen_tokens in config.py."
                )
        if temperature is None or temperature <= 0:
            raise ValueError(
                f"generate(): temperature must be > 0, got {temperature!r}. "
                f"For greedy decoding use top_k=1 with a small temperature."
            )
        # BUGFIX: @torch.no_grad() does NOT put the module in eval mode, so
        # sampling from a model left in .train() ran with dropout active (and
        # kept mutating MoE expert_bias). Force eval, restore afterwards.
        was_training = self.training
        self.eval()
        try:
            return self._generate_loop(idx, max_new_tokens, temperature, top_k, use_cache)
        finally:
            if was_training:
                self.train()

    def _generate_loop(self, idx, max_new_tokens, temperature, top_k, use_cache=True):
        block_size = self.config.block_size
        cached = bool(use_cache) and self.supports_kv_cache
        cache = None
        for _ in range(max_new_tokens):
            if cached and idx.size(1) <= block_size:
                # Full context still fits: KV cache gives the same result as re-encoding it.
                if cache is None:
                    cache = self.new_kv_cache()
                    logits, _ = self(idx, kv_cache=cache)         # prefill
                else:
                    logits, _ = self(idx[:, -1:], kv_cache=cache)  # one token
            else:
                # Past block_size the original sliding-window re-encode applies
                # (positions re-based each step), which a cache can't reproduce.
                cached, cache = False, None
                idx_cond = idx if idx.size(1) <= block_size else idx[:, -block_size:]
                logits, _ = self(idx_cond)
            
            
            logits = logits[:, -1, :] / temperature  
            
            
            if top_k is not None:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits < v[:, [-1]]] = -float('Inf')
            
            
            probs = F.softmax(logits.float(), dim=-1)  # (B, vocab_size)
            
            # Sample next token
            idx_next = torch.multinomial(probs, num_samples=1)  # (B, 1)
            
            # Append to sequence
            idx = torch.cat((idx, idx_next), dim=1)  # (B, T+1)

        return idx