phanerozoic commited on Jul 17

Commit

7137be6

verified ·

1 Parent(s): 4611d12

Uploaded using `kernel-builder`.

Browse files

Files changed (46) hide show

.gitattributes +9 -0
build/torch211-cxx11-cu126-x86_64-linux/__init__.py +863 -0
build/torch211-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch211-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu126-x86_64-linux/metadata.json +36 -0
build/torch211-cxx11-cu126-x86_64-linux/metakernel/__init__.py +26 -0
build/torch211-cxx11-cu128-x86_64-linux/__init__.py +863 -0
build/torch211-cxx11-cu128-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu128-x86_64-linux/metadata.json +38 -0
build/torch211-cxx11-cu128-x86_64-linux/metakernel/__init__.py +26 -0
build/torch211-cxx11-cu130-x86_64-linux/__init__.py +863 -0
build/torch211-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch211-cxx11-cu130-x86_64-linux/metadata.json +38 -0
build/torch211-cxx11-cu130-x86_64-linux/metakernel/__init__.py +26 -0
build/torch212-cxx11-cu126-x86_64-linux/__init__.py +863 -0
build/torch212-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch212-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu126-x86_64-linux/metadata.json +36 -0
build/torch212-cxx11-cu126-x86_64-linux/metakernel/__init__.py +26 -0
build/torch212-cxx11-cu130-x86_64-linux/__init__.py +863 -0
build/torch212-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu130-x86_64-linux/metadata.json +38 -0
build/torch212-cxx11-cu130-x86_64-linux/metakernel/__init__.py +26 -0
build/torch212-cxx11-cu132-x86_64-linux/__init__.py +863 -0
build/torch212-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
build/torch212-cxx11-cu132-x86_64-linux/metadata.json +38 -0
build/torch212-cxx11-cu132-x86_64-linux/metakernel/__init__.py +26 -0
build/torch213-cxx11-cu126-x86_64-linux/__init__.py +863 -0
build/torch213-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch213-cxx11-cu126-x86_64-linux/_ops.py +9 -0
build/torch213-cxx11-cu126-x86_64-linux/metadata.json +36 -0
build/torch213-cxx11-cu126-x86_64-linux/metakernel/__init__.py +26 -0
build/torch213-cxx11-cu130-x86_64-linux/__init__.py +863 -0
build/torch213-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch213-cxx11-cu130-x86_64-linux/_ops.py +9 -0
build/torch213-cxx11-cu130-x86_64-linux/metadata.json +38 -0
build/torch213-cxx11-cu130-x86_64-linux/metakernel/__init__.py +26 -0
build/torch213-cxx11-cu132-x86_64-linux/__init__.py +863 -0
build/torch213-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so +3 -0
build/torch213-cxx11-cu132-x86_64-linux/_ops.py +9 -0
build/torch213-cxx11-cu132-x86_64-linux/metadata.json +38 -0
build/torch213-cxx11-cu132-x86_64-linux/metakernel/__init__.py +26 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu128-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch211-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch212-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text
+build/torch213-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so filter=lfs diff=lfs merge=lfs -text

build/torch211-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch211-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:474c2f8fa87fd25a548c07014c46827295f5b2eace19b7e190f0090fa61e8877
+size 626176

build/torch211-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch211-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,36 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "R0wvj6h/0lpUjAcBTEaCcpX1surOGbfhkPAJD6YeiHc=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch211-cxx11-cu126-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch211-cxx11-cu128-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:741d3ad6ca34a13caade26ca26546a48c2a91c819c197dbb10e98336b5470d98
+size 941440

build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "dB061so0oTyq3ibKJlRqSMKpHIGcGX27EOmDNrVHDZg=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch211-cxx11-cu128-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch211-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:b2df8ca3e350475a7f0aebd96c8820a08c92a203bc890aa26182e7c4b8234386
+size 961480

build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "st+Mo+NQR1p/CuvZbIggoIySogO8iQqiYYLnxLgjQ4Y=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch211-cxx11-cu130-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch212-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:95a7ad4c20e35be975f172307ed4ccfdfaacda544ffded4a0394a6c66451920c
+size 636184

build/torch212-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch212-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,36 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "laetTCDjW+l18XIwftTM/fqs2lRP/e1KA5SmxmRRkgw=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch212-cxx11-cu126-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch212-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c1808009535a071b8ee0439dafe12d3b6a84dc9be242f5c700957cffb9b26670
+size 967576

build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "wYCACVNaBxuO4EOdr+EtO2qE3JviQvXHAJV8/7myZnA=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch212-cxx11-cu130-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch212-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:e88576465f9b2bebeea6c417bf42418137f71e9029cbf52dc03e83f86e433a6d
+size 980192

build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "6IV2Rl+bK+vupsQXv0JBgTf3HpApy/UtwD6D+G5DOm0=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch212-cxx11-cu132-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch213-cxx11-cu126-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch213-cxx11-cu126-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:97f07e5ff1283cbc7f715f1b6dda828cd79f4164e9a1a838f33b8c36889f0901
+size 636184

build/torch213-cxx11-cu126-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch213-cxx11-cu126-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,36 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "l/B+X/EoPLx/cV8bbdqCjNefQWTpoag48zuMNoifCQE=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cu126-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch213-cxx11-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch213-cxx11-cu130-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7b1c2a4f6a13c1ca3458133df0c62730801fb87f38a0eeaa556752c2eb87c25a
+size 967576

build/torch213-cxx11-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch213-cxx11-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "exwqT2oTwco0WBM98MYnMIAfuH84oO6qVWdSwuuHwlo=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cu130-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch213-cxx11-cu132-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,863 @@

+"""metakernel: the measuring instrument for the kernel-design loop.
+An agent iterating on CUDA candidates is good at writing code and bad at
+trusting its own measurements; it needs trustworthy timing, adversarial
+correctness comparison, and measured facts about the device it targets.
+metakernel ships those as loadable ops:
+    from kernels import get_kernel
+    mk = get_kernel("phanerozoic/metakernel", version=1,
+                    trust_remote_code=True)
+    dossier = mk.probe_device()                       # measured facts
+    report  = mk.bench(lambda: op(x), iters=200)      # throttle-rejected
+    verdict = mk.compare(ref, cand, gen)              # ULP evidence
+    table   = mk.sweep(factory, {"BLOCK": [64, 128]}) # config search
+Every op returns plain dicts and tensors, never prose. Every dossier entry
+is produced by a dedicated probe kernel rather than a spec sheet, carries
+the SM clock it was measured under, and labels throttled samples instead
+of letting them poison medians. No hardware counters are used: everything
+is inferred from timing, clocks, and the occupancy API, so entries carry a
+confidence field (timing-differential inference can misattribute).
+"""
+import math
+import time
+import torch
+from ._ops import ops
+__all__ = ["probe_device", "bench", "compare", "order_shuffles",
+           "fuzz_shapes", "sweep", "alloc_stamps", "read_stamps",
+           "measure_clock", "ulp_diff", "ops"]
+_MMA_KINDS = {
+    "fp16": (0, 16 * 16 * 16),
+    "bf16": (1, 16 * 16 * 16),
+    "tf32": (2, 16 * 16 * 8),
+    "int8": (3, 16 * 16 * 16),
+    "fp8_e4m3": (4, 16 * 8 * 32),
+}
+def _sync():
+    torch.cuda.synchronize()
+def _events(fn, reps):
+    """Time reps calls of fn with one event pair each; returns ms list."""
+    starts = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    stops = [torch.cuda.Event(enable_timing=True) for _ in range(reps)]
+    for s, e in zip(starts, stops):
+        s.record()
+        fn()
+        e.record()
+    _sync()
+    return [s.elapsed_time(e) for s, e in zip(starts, stops)]
+def measure_clock(device="cuda", ticks=4_000_000, blocks=None):
+    """Effective SM clock in GHz under an FMA spin of `ticks` clock64 ticks.
+    Wall time over elapsed ticks is the frequency the ticks ran at."""
+    props = torch.cuda.get_device_properties(device)
+    if blocks is None:
+        blocks = props.multi_processor_count
+    out = torch.zeros(2, dtype=torch.int64, device=device)
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    ops.mk_spin(blocks, 256, ticks, out, sink)  # warm
+    start = torch.cuda.Event(enable_timing=True)
+    stop = torch.cuda.Event(enable_timing=True)
+    start.record()
+    ops.mk_spin(blocks, 256, ticks, out, sink)
+    stop.record()
+    _sync()
+    ms = start.elapsed_time(stop)
+    elapsed = int(out[0].item())
+    return elapsed / (ms * 1e6)
+def _median(xs):
+    s = sorted(xs)
+    n = len(s)
+    return s[n // 2] if n % 2 else 0.5 * (s[n // 2 - 1] + s[n // 2])
+def _entry(value, units, method, clock_ghz, confidence, note=""):
+    d = {"value": value, "units": units, "method": method,
+         "clock_ghz": round(clock_ghz, 3), "confidence": confidence}
+    if note:
+        d["note"] = note
+    return d
+def _sattolo_ring(n, device, generator=None):
+    """A random single-cycle permutation as an int32 next-pointer table, so
+    a pointer chase visits every slot before repeating."""
+    perm = torch.randperm(n, generator=generator)
+    ring = torch.empty(n, dtype=torch.int32)
+    ring[perm] = torch.roll(perm, -1).to(torch.int32)
+    return ring.to(device)
+def probe_device(device="cuda", quick=False):
+    """Empirical device dossier. Each entry is measured by a dedicated
+    probe kernel and carries units, method, the SM clock it ran under, and
+    a confidence field."""
+    props = torch.cuda.get_device_properties(device)
+    sm = props.multi_processor_count
+    l2 = props.L2_cache_size
+    d = {
+        "device": {
+            "name": props.name,
+            "sm_arch": f"sm{props.major}{props.minor}",
+            "sms": sm,
+            "l2_bytes": l2,
+            "vram_bytes": props.total_memory,
+            "torch": torch.__version__,
+            "cuda_runtime": torch.version.cuda,
+        }
+    }
+    sink = torch.zeros(2, dtype=torch.float32, device=device)
+    out64 = torch.zeros(2, dtype=torch.int64, device=device)
+    reps = 3 if quick else 5
+    # ---- clock under sustained load
+    traj = []
+    windows = 8 if quick else 24
+    for _ in range(windows):
+        traj.append(measure_clock(device, ticks=30_000_000))
+    d["clock"] = {
+        "max_ghz": round(max(traj), 3),
+        "min_ghz": round(min(traj), 3),
+        "final_ghz": round(traj[-1], 3),
+        "sag_pct": round(100 * (1 - traj[-1] / max(traj)), 1),
+        "units": "GHz",
+        "method": f"{windows} back-to-back FMA spin windows of 30M ticks; "
+                  "wall/ticks per window",
+        "confidence": "high",
+    }
+    clk = max(traj)
+    # ---- HBM bandwidth: triad per access width, plus read-only,
+    # write-only, and random-gather
+    n4 = 16 << 20  # 16M float4 = 256 MB per buffer
+    if props.total_memory < 4 << 30:
+        n4 = 4 << 20
+    a = torch.rand(n4 * 4, device=device)
+    b = torch.rand(n4 * 4, device=device)
+    c = torch.empty(n4 * 4, device=device)
+    hbm = {}
+    for width in (1, 2, 4):
+        ops.mk_triad(a, b, c, width, 1.5)  # warm
+        ms = _median(_events(lambda w=width: ops.mk_triad(a, b, c, w, 1.5),
+                             reps))
+        gbs = 3 * a.numel() * 4 / (ms * 1e-3) / 1e9
+        hbm[f"triad_{4 * width}B"] = round(gbs, 1)
+    ops.mk_read(a, sink)
+    ms = _median(_events(lambda: ops.mk_read(a, sink), reps))
+    hbm["read_only"] = round(a.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    ops.mk_write(c, 1.0)
+    ms = _median(_events(lambda: ops.mk_write(c, 1.0), reps))
+    hbm["write_only"] = round(c.numel() * 4 / (ms * 1e-3) / 1e9, 1)
+    g = torch.Generator(device="cpu").manual_seed(7)
+    idx = torch.randint(0, n4, (n4,), generator=g,
+                        dtype=torch.int32).to(device)
+    ops.mk_gather(a, idx, c, 256, 0)
+    ms = _median(_events(lambda: ops.mk_gather(a, idx, c, 256, 0), reps))
+    # traffic accounting: idx reads + 16B random loads + streamed stores
+    hbm["random_gather_16B"] = round(
+        (idx.numel() * 4 + 2 * n4 * 16) / (ms * 1e-3) / 1e9, 1)
+    d["hbm_bw"] = _entry(hbm, "GB/s", "streaming triad c=a+s*b per access "
+                         "width; read-only fold; write-only fill; random "
+                         "float4 gather (16B granules, nominal traffic)",
+                         clk, "high",
+                         "gather moves full cache lines per 16B granule, "
+                         "so its nominal number is the achievable rate for "
+                         "random access, not a line-efficiency claim")
+    # ---- L2 bandwidth (working set 3 buffers inside L2)
+    ws = max(l2 // 4, 1 << 20)
+    nl2 = ws // 16  # float4 elems per buffer
+    al = a[:nl2 * 4].contiguous()
+    bl = b[:nl2 * 4].contiguous()
+    cl = c[:nl2 * 4].contiguous()
+    passes = 64
+    ops.mk_triad_passes(al, bl, cl, 1.5, passes)
+    ms = _median(_events(
+        lambda: ops.mk_triad_passes(al, bl, cl, 1.5, passes), reps))
+    gbs = 3 * al.numel() * 4 * passes / (ms * 1e-3) / 1e9
+    d["l2_bw"] = _entry(round(gbs, 1), "GB/s",
+                        f"in-kernel repeated triad passes over "
+                        f"{3 * ws >> 20} MB resident working set", clk,
+                        "medium", "first pass streams from HBM; amortized "
+                        f"over {passes} passes")
+    # ---- latencies: pointer chase per level. Footprints are chosen so
+    # the chain hits the intended level: the L2 ring (2 MB) exceeds L1,
+    # fits L2 on every supported part, and spans few enough pages to stay
+    # TLB-resident after a full-cycle warm walk; the HBM chain is long
+    # enough (2x L2 in touched lines) that it evicts itself and cannot be
+    # served from cache.
+    lat = {}
+    idle_clk = measure_clock(device, ticks=2_000_000, blocks=1)
+    ring_s = _sattolo_ring(8192, device)
+    ops.mk_chase_shared(ring_s, 1 << 14, out64)
+    _sync()
+    hops = 1 << 16
+    ops.mk_chase_shared(ring_s, hops, out64)
+    _sync()
+    lat["smem_cycles"] = round(int(out64[0].item()) / hops, 1)
+    l1_ring_n = (16 << 10) // 4  # 16 KB: L1-resident after a warm walk
+    ring_l1 = _sattolo_ring(l1_ring_n, device)
+    ops.mk_chase_global(ring_l1, l1_ring_n, out64)
+    _sync()
+    hops_l1 = 1 << 15
+    ops.mk_chase_global(ring_l1, hops_l1, out64)
+    _sync()
+    lat["l1_cycles"] = round(int(out64[0].item()) / hops_l1, 1)
+    l2_ring_n = (2 << 20) // 4
+    ring_l2 = _sattolo_ring(l2_ring_n, device)
+    ops.mk_chase_global(ring_l2, l2_ring_n, out64)  # full-cycle warm walk
+    _sync()
+    hops_l2 = 1 << 14
+    ops.mk_chase_global(ring_l2, hops_l2, out64)
+    _sync()
+    lat["l2_cycles"] = round(int(out64[0].item()) / hops_l2, 1)
+    hbm_bytes = min(max(4 * l2, 64 << 20), props.total_memory // 8)
+    ring_h = _sattolo_ring(int(hbm_bytes) // 4, device)
+    hops_h = max((2 * l2) // 128, 1 << 17)
+    ops.mk_chase_global(ring_h, 1 << 14, out64)  # fault pages / warm TLB
+    _sync()
+    ops.mk_chase_global(ring_h, hops_h, out64)
+    _sync()
+    lat["hbm_cycles"] = round(int(out64[0].item()) / hops_h, 1)
+    lat["ns_per_cycle"] = round(1.0 / idle_clk, 3)
+    d["latency"] = _entry(lat, "cycles (single-lane dependent loads)",
+                          "Sattolo-cycle pointer chase; 16 KB ring for L1, "
+                          "2 MB TLB-resident ring for L2, self-evicting "
+                          "chain (2x L2 in touched lines) for HBM",
+                          idle_clk, "high",
+                          "smem includes index arithmetic; HBM includes "
+                          "residual TLB effects at large footprints")
+    del ring_h, ring_l2, ring_l1, ring_s
+    # ---- fp32 FMA and tensor-core throughput
+    blocks = sm * 4
+    iters = 20_000 if quick else 60_000
+    ops.mk_fma_f32(blocks, 256, iters, sink)
+    ms = _median(_events(lambda: ops.mk_fma_f32(blocks, 256, iters, sink),
+                         reps))
+    flops = blocks * 256 * iters * 16 * 2 * 2  # 2 chains x 2 flops x unroll
+    d["fma_f32"] = _entry(round(flops / (ms * 1e-3) / 1e12, 2), "TFLOP/s",
+                          "register-resident FMA chains", clk, "high")
+    sink64 = torch.zeros(2, dtype=torch.float64, device=device)
+    it64 = max(iters // 16, 1000)
+    ops.mk_fma_f64(blocks, 256, it64, sink64)
+    ms = _median(_events(
+        lambda: ops.mk_fma_f64(blocks, 256, it64, sink64), reps))
+    flops64 = blocks * 256 * it64 * 16 * 2 * 2
+    d["fma_f64"] = _entry(round(flops64 / (ms * 1e-3) / 1e12, 3), "TFLOP/s",
+                          "register-resident double FMA chains (the DFMA "
+                          "pipe)", clk, "high")
+    mma = {}
+    for name, (kind, mnk) in _MMA_KINDS.items():
+        it = 30_000 if quick else 100_000
+        r = ops.mk_mma(kind, blocks, 8, 512, sink)
+        if r != 0:
+            mma[name] = "unsupported"
+            continue
+        _sync()
+        ms = _median(_events(
+            lambda k=kind, i=it: ops.mk_mma(k, blocks, 8, i, sink), reps))
+        flops = blocks * 8 * it * 4 * 2 * mnk
+        mma[name] = round(flops / (ms * 1e-3) / 1e12, 1)
+    d["mma"] = _entry(mma, "TFLOP/s (TOP/s for int8)",
+                      "register-fragment mma_sync loops (wmma; fp8 via "
+                      "inline PTX m16n8k32)", clk, "medium",
+                      "dense-tile rate without memory traffic; real "
+                      "kernels see this only when staging hides behind it")
+    # ---- atomic contention curve: throughput vs number of distinct
+    # addresses the whole grid hammers (slots = 1 is full contention;
+    # slots = threads is contention-free), plus the shared-memory point
+    n_thr = blocks * 256
+    buf = torch.zeros(n_thr, dtype=torch.float32, device=device)
+    at = {"curve_slots": {}}
+    it_at = 2_000 if quick else 6_000
+    slot_points = [1, 8, 64, 512, 4096, n_thr]
+    for slots in slot_points:
+        buf.zero_()
+        ops.mk_atomics(1, blocks, 256, it_at, buf, slots)
+        ms = _median(_events(
+            lambda s=slots: ops.mk_atomics(1, blocks, 256, it_at, buf, s),
+            reps))
+        at["curve_slots"][str(slots)] = round(
+            n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    ops.mk_atomics(3, blocks, 256, it_at, buf, 1)
+    ms = _median(_events(
+        lambda: ops.mk_atomics(3, blocks, 256, it_at, buf, 1), reps))
+    at["shared_same_addr"] = round(n_thr * it_at / (ms * 1e-3) / 1e9, 2)
+    d["atomics_f32"] = _entry(at, "Gop/s",
+                              "atomicAdd hammer; curve sweeps the number "
+                              "of distinct global addresses from 1 (full "
+                              "contention) to one per thread", clk, "high")
+    # ---- launch overhead
+    anchor = sink
+    n_launch = 500 if quick else 2000
+    _sync()
+    t0 = time.perf_counter()
+    for _ in range(n_launch):
+        ops.mk_empty(anchor)
+    _sync()
+    queued = (time.perf_counter() - t0) / n_launch * 1e6
+    n_rt = 100 if quick else 300
+    t0 = time.perf_counter()
+    for _ in range(n_rt):
+        ops.mk_empty(anchor)
+        _sync()
+    rt = (time.perf_counter() - t0) / n_rt * 1e6
+    d["launch_overhead"] = _entry(
+        {"queued_us": round(queued, 2), "round_trip_us": round(rt, 2)},
+        "us", "empty-kernel trains, queued vs launch+sync", clk, "high",
+        "host-dependent: includes torch op dispatch")
+    # ---- grid barrier
+    bar_iters = 500 if quick else 2000
+    r = ops.mk_barrier(sm, 256, bar_iters, out64)
+    if r == 0:
+        _sync()
+        ticks = int(out64[0].item())
+        d["grid_barrier"] = _entry(
+            round(ticks / bar_iters / clk / 1000.0, 2), "us per grid.sync",
+            f"cooperative kernel, {sm} blocks x 256 threads", clk, "high")
+    else:
+        d["grid_barrier"] = _entry("unsupported", "", "cooperative launch "
+                                   "unavailable or grid too large", clk,
+                                   "high")
+    # ---- occupancy table (API) plus timed knees: the same random gather
+    # at every (threads, dyn_smem) point; when the forced blocks/SM stop
+    # covering memory latency, achieved bandwidth falls off the cliff
+    occ = {}
+    for threads in (128, 256, 512, 1024):
+        row = {}
+        for smem_kb in (0, 16, 32, 48):
+            row[f"{smem_kb}KB"] = int(ops.mk_occupancy(
+                threads, smem_kb * 1024, anchor))
+        occ[f"t{threads}"] = row
+    d["occupancy_blocks_per_sm"] = _entry(
+        occ, "blocks/SM", "occupancy API on the float4 triad kernel",
+        clk, "high", "API-derived, not timing-derived")
+    knee_iters = 6_000 if quick else 20_000
+    knee_blocks = sm * 8
+    knees = {}
+    for threads in (128, 256, 1024):
+        row = {}
+        for smem_kb in (0, 32, 48):
+            ops.mk_fma_dep(knee_blocks, threads, smem_kb * 1024,
+                           knee_iters, sink)
+            ms = _median(_events(
+                lambda t=threads, s=smem_kb: ops.mk_fma_dep(
+                    knee_blocks, t, s * 1024, knee_iters, sink), reps))
+            fl = knee_blocks * threads * knee_iters * 16 * 2
+            row[f"{smem_kb}KB"] = round(fl / (ms * 1e-3) / 1e12, 2)
+        knees[f"t{threads}"] = row
+    d["occupancy_knees_fma_tflops"] = _entry(
+        knees, "TFLOP/s", "a single dependent FMA chain per thread, timed "
+        "at forced (threads, dyn smem) occupancies over fixed total work; "
+        "throughput is proportional to resident threads until the pipe "
+        "saturates, so the cliff marks where occupancy stops covering "
+        "latency", clk, "high",
+        "timing-derived counterpart to the API table above")
+    return d
+def bench(fn, iters=100, warmup=10, bytes=None, flops=None, dossier=None,
+          dtype="fp16", clock_guard=0.97, clock_stride=8, graph=False,
+          device="cuda"):
+    """Median-of-N cudaEvent timing with throttle rejection.
+    The SM clock is probed every `clock_stride` samples; samples inside a
+    window whose clock sagged below `clock_guard` x the best observed
+    clock are rejected and counted rather than silently averaged. With
+    declared `bytes`/`flops` per call, reports achieved GB/s / TFLOP/s;
+    with a `dossier`, adds roofline placement against the measured peaks.
+    `graph=True` captures fn() into a CUDA graph and times graph replays,
+    stripping Python and torch dispatch from the measurement: the right
+    mode for kernels in the few-microsecond range, where dispatch is
+    otherwise the thing being measured. Capture requires fn to be
+    graph-safe (no sync, no allocation growth, static shapes)."""
+    for _ in range(warmup):
+        fn()
+    _sync()
+    if graph:
+        g = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(g):
+            fn()
+        fn = g.replay
+        for _ in range(warmup):
+            fn()
+        _sync()
+    samples = []
+    clocks = []
+    stride = clock_stride
+    for base in range(0, iters, stride):
+        clocks.append(measure_clock(device, ticks=1_500_000))
+        n = min(stride, iters - base)
+        samples.append((_events(fn, n), len(clocks) - 1))
+    clocks.append(measure_clock(device, ticks=1_500_000))
+    best_clk = max(clocks)
+    kept, rejected = [], 0
+    flat = []
+    for group, ci in samples:
+        window_ok = (clocks[ci] >= clock_guard * best_clk and
+                     clocks[min(ci + 1, len(clocks) - 1)] >=
+                     clock_guard * best_clk)
+        for ms in group:
+            flat.append(ms)
+            if window_ok:
+                kept.append(ms)
+            else:
+                rejected += 1
+    if not kept:
+        kept = flat
+    med = _median(kept)
+    rep = {
+        "median_ms": round(med, 4),
+        "min_ms": round(min(kept), 4),
+        "mean_ms": round(sum(kept) / len(kept), 4),
+        "samples": len(flat),
+        "rejected_throttled": rejected,
+        "graph_captured": bool(graph),
+        "clock_ghz": {"best": round(best_clk, 3),
+                      "worst": round(min(clocks), 3)},
+    }
+    if bytes is not None:
+        rep["achieved_gbs"] = round(bytes / (med * 1e-3) / 1e9, 1)
+    if flops is not None:
+        rep["achieved_tflops"] = round(flops / (med * 1e-3) / 1e12, 2)
+    if bytes and flops:
+        rep["arithmetic_intensity"] = round(flops / bytes, 2)
+    if dossier is not None and bytes and flops:
+        bw = max(dossier["hbm_bw"]["value"].values()) * 1e9
+        mma_v = dossier["mma"]["value"].get(dtype)
+        peak = (mma_v if isinstance(mma_v, (int, float))
+                else dossier["fma_f32"]["value"]) * 1e12
+        ai = flops / bytes
+        roof = min(peak, ai * bw)
+        rep["roofline"] = {
+            "bound": "memory" if ai * bw < peak else "compute",
+            "ridge_flops_per_byte": round(peak / bw, 1),
+            "pct_of_roof": round(100 * (flops / (med * 1e-3)) / roof, 1),
+            "peak_dtype": dtype,
+        }
+    return rep
+def _to_ordered_int(t):
+    """Map float bit patterns to integers whose difference is the ULP
+    distance: positive floats keep their bits, negative floats mirror
+    below zero, so the mapping is monotone in the float value."""
+    if t.dtype == torch.float32:
+        u = t.view(torch.int32).to(torch.int64) & 0xFFFFFFFF
+        half = 1 << 31
+    elif t.dtype in (torch.float16, torch.bfloat16):
+        u = t.view(torch.int16).to(torch.int64) & 0xFFFF
+        half = 1 << 15
+    elif t.dtype == torch.float64:
+        i = t.view(torch.int64)
+        return torch.where(i >= 0, i, (-(1 << 63)) - i)
+    else:
+        raise ValueError(f"unsupported dtype {t.dtype}")
+    return torch.where(u < half, u, half - u)
+def ulp_diff(a, b):
+    """Elementwise ULP distance between same-dtype float tensors. NaNs in
+    the same position count as 0; a one-sided NaN counts as 2**62."""
+    assert a.dtype == b.dtype and a.shape == b.shape
+    ia, ib = _to_ordered_int(a), _to_ordered_int(b)
+    d = (ia - ib).abs()
+    both_nan = torch.isnan(a) & torch.isnan(b)
+    one_nan = torch.isnan(a) ^ torch.isnan(b)
+    d = torch.where(both_nan, torch.zeros_like(d), d)
+    d = torch.where(one_nan, torch.full_like(d, 1 << 62), d)
+    return d
+class order_shuffles:
+    """Built-in order shuffles for the common contractual-noise axes."""
+    @staticmethod
+    def last_dim():
+        """Permute the last dim of every tensor arg identically: the
+        accumulation order of a last-dim reduction changes, the value
+        does not."""
+        def shuffle(args, seed):
+            g = torch.Generator(device=args[0].device).manual_seed(seed)
+            p = torch.randperm(args[0].shape[-1], device=args[0].device,
+                               generator=g)
+            return tuple(x[..., p] for x in args)
+        return shuffle
+    @staticmethod
+    def matmul_k():
+        """Permute K of (A [.., M, K], B [.., K, N]) consistently: the
+        product is unchanged, the accumulation order is not."""
+        def shuffle(args, seed):
+            a_m, b_m = args[0], args[1]
+            g = torch.Generator(device=a_m.device).manual_seed(seed)
+            p = torch.randperm(a_m.shape[-1], device=a_m.device,
+                               generator=g)
+            return (a_m[..., p], b_m[..., p, :]) + tuple(args[2:])
+        return shuffle
+def _as_outputs(x):
+    return list(x) if isinstance(x, (tuple, list)) else [x]
+def compare(ref, cand, gen, trials=8, order_shuffle=None, order_trials=8,
+            ulp_tol=None, atol=None, rtol=None, band_margin=4.0, seed=0):
+    """Adversarial correctness comparison with ULP evidence.
+    `gen(seed) -> args` builds inputs; `ref(*args)` and `cand(*args)`
+    return a tensor or a tuple of tensors (each output is compared and
+    the worst governs). If `order_shuffle(args, seed) -> args` is given
+    and reorders contractual-noise dimensions (see `order_shuffles` for
+    the common cases), the reference is treated as a distribution:
+    candidates are judged against the measured reduction-order band
+    rather than a point. Verdict, stated in the report: max ULP <=
+    band_margin * band + 2 when a band is measured, else max ULP <=
+    ulp_tol (default 2); if `atol`/`rtol` are given they are an
+    alternative acceptance route, checked as |c - r| <= atol + rtol|r|."""
+    worst = {"max_ulp": -1}
+    total_hist = {}
+    per_trial = []
+    max_abs_seen = 0.0
+    max_rel_seen = 0.0
+    for t in range(trials):
+        args = gen(seed + t)
+        r_outs = _as_outputs(ref(*args))
+        c_outs = _as_outputs(cand(*args))
+        assert len(r_outs) == len(c_outs), "output arity mismatch"
+        # the order band is POINTWISE: a near-zero reduction output is
+        # legitimately thousands of ULP order-sensitive (cancellation)
+        # while its neighbors are stable, so each element is judged
+        # against its own measured sensitivity, not a global max
+        bands = None
+        band = 0
+        band_score = 0.0
+        if order_shuffle is not None:
+            bands = [torch.zeros_like(ro, dtype=torch.int64)
+                     for ro in r_outs]
+            for k in range(order_trials):
+                r2 = _as_outputs(
+                    ref(*order_shuffle(args, 1000 + seed + t * 131 + k)))
+                for bi, (ro, r2o) in enumerate(zip(r_outs, r2)):
+                    bands[bi] = torch.maximum(bands[bi],
+                                              ulp_diff(ro, r2o))
+            band = max(int(b.max().item()) if b.numel() else 0
+                       for b in bands)
+        max_ulp = 0
+        mean_ulp = 0.0
+        first_div = -1
+        n_el = 0
+        nan_r = nan_c = inf_r = inf_c = 0
+        for oi, (r, c) in enumerate(zip(r_outs, c_outs)):
+            d = ulp_diff(r, c)
+            if bands is not None and d.numel():
+                ratio = d.double() / (bands[oi].double() + 2.0)
+                band_score = max(band_score, float(ratio.max().item()))
+            if d.numel():
+                max_ulp = max(max_ulp, int(d.max().item()))
+                mean_ulp += float(d.double().sum().item())
+                n_el += d.numel()
+                nz = (d > 0)
+                if first_div < 0 and nz.any():
+                    first_div = int(nz.flatten().nonzero()[0].item())
+                if nz.any():
+                    buckets = torch.clamp(
+                        torch.log2(d[nz].double()).floor(), 0, 62
+                    ).to(torch.int64)
+                    counts = torch.bincount(buckets, minlength=63)
+                    for bkt in counts.nonzero().flatten().tolist():
+                        total_hist[bkt] = total_hist.get(bkt, 0) + int(
+                            counts[bkt].item())
+                fin = torch.isfinite(r) & torch.isfinite(c)
+                if fin.any():
+                    diff = (c[fin].double() - r[fin].double()).abs()
+                    max_abs_seen = max(max_abs_seen,
+                                       float(diff.max().item()))
+                    rel = diff / r[fin].double().abs().clamp(min=1e-300)
+                    max_rel_seen = max(max_rel_seen,
+                                       float(rel.max().item()))
+            nan_r += int(torch.isnan(r).sum().item())
+            nan_c += int(torch.isnan(c).sum().item())
+            inf_r += int(torch.isinf(r).sum().item())
+            inf_c += int(torch.isinf(c).sum().item())
+        trial_rep = {
+            "max_ulp": max_ulp,
+            "mean_ulp": round(mean_ulp / max(n_el, 1), 3),
+            "first_divergent_index": first_div,
+            "nan_ref": nan_r, "nan_cand": nan_c,
+            "inf_ref": inf_r, "inf_cand": inf_c,
+            "order_band_ulp": band,
+            "band_score": round(band_score, 2),
+        }
+        per_trial.append(trial_rep)
+        if max_ulp > worst["max_ulp"]:
+            worst = trial_rep
+    band_max = max(t["order_band_ulp"] for t in per_trial)
+    max_seen = max(t["max_ulp"] for t in per_trial)
+    if order_shuffle is not None:
+        score = max(t["band_score"] for t in per_trial)
+        ulp_ok = score <= band_margin
+        criterion = (f"pointwise max(ulp / (order_band + 2)) = "
+                     f"{score:.2f} <= band_margin {band_margin}")
+    else:
+        tol = ulp_tol if ulp_tol is not None else 2
+        ulp_ok = max_seen <= tol
+        criterion = f"max_ulp {max_seen} <= ulp_tol {tol}"
+    tol_ok = False
+    if atol is not None or rtol is not None:
+        tol_ok = max_abs_seen <= (atol or 0) + (rtol or 0) * 1.0 or \
+            max_rel_seen <= (rtol or 0) or max_abs_seen <= (atol or 0)
+        criterion += (f"; OR max_abs {max_abs_seen:.3e} <= atol {atol} / "
+                      f"max_rel {max_rel_seen:.3e} <= rtol {rtol}")
+    return {
+        "pass": bool(ulp_ok or tol_ok),
+        "criterion": criterion,
+        "worst_trial": worst,
+        "max_abs": max_abs_seen,
+        "max_rel": max_rel_seen,
+        "trials": per_trial,
+        "ulp_hist_log2": dict(sorted(total_hist.items())),
+    }
+_FUZZ_VARIANTS = ("contig", "transposed", "strided_rows", "offset_slice",
+                  "overflow_expand")
+def _context_alive(device):
+    try:
+        t = torch.ones(4, device=device)
+        torch.cuda.synchronize()
+        return bool(t.sum().item() == 4.0)
+    except Exception:  # noqa: BLE001
+        return False
+def fuzz_shapes(op, ref, make, shapes=None, variants=_FUZZ_VARIANTS,
+                dtypes=(torch.float32,), ulp_tol=64, device="cuda"):
+    """The ugly envelope: run op vs ref across edge shapes, dtypes, and
+    layout mutations; report ok / mismatch / exception per case.
+    `make(shape, dtype)` (or `make(shape)`) builds a base input, either a
+    tensor or a tuple of tensors for multi-argument ops; layout variants
+    derive non-contiguous and misaligned views whose CONTENT equals a
+    contiguous copy, so `op(*xs)` must match `ref(*xs)` on every variant.
+    `overflow_expand` stride-0-expands the input past 2**31 logical
+    elements to catch int32 indexing, and is skipped with status
+    `skipped_vram` when materializing the output would not fit. After any
+    exception the CUDA context is health-checked; if a candidate killed
+    it, remaining cases are marked `context_lost` instead of producing
+    garbage."""
+    if shapes is None:
+        shapes = [(0, 8), (1, 1), (1, 17), (3, 63), (17, 129), (128, 128),
+                  (1023, 5), (2, 4096)]
+    def build(shape, dtype):
+        try:
+            out = make(tuple(shape), dtype)
+        except TypeError:
+            out = make(tuple(shape))
+        return out if isinstance(out, (tuple, list)) else (out,)
+    def variant_of(x, v):
+        if v == "contig":
+            return x
+        if v == "transposed":
+            return x.t() if x.dim() == 2 else x
+        if v == "strided_rows":
+            return x[::2] if x.shape[0] >= 2 else x
+        if v == "offset_slice":
+            # a storage-offset-1 view: misaligned for vectorized loads
+            return x.flatten()[1:] if x.numel() > 1 else x
+        if v == "overflow_expand":
+            if x.numel() == 0:
+                return None
+            per = max(x.numel(), 1)
+            rep = (1 << 31) // per + 2
+            return x.reshape((1,) + tuple(x.shape)).expand(
+                (rep,) + tuple(x.shape))
+        return x
+    rows = []
+    lost = False
+    for dtype in dtypes:
+        for shape in shapes:
+            bases = build(shape, dtype)
+            for v in variants:
+                case = {"shape": tuple(shape), "variant": v,
+                        "dtype": str(dtype).replace("torch.", "")}
+                if lost:
+                    case["status"] = "context_lost"
+                    rows.append(case)
+                    continue
+                xs = [variant_of(b, v) for b in bases]
+                if any(x is None for x in xs):
+                    case["status"] = "skipped_empty"
+                    rows.append(case)
+                    continue
+                if v == "overflow_expand":
+                    logical = max(x.numel() for x in xs)
+                    need = logical * xs[0].element_size() * 2
+                    free, _total = torch.cuda.mem_get_info()
+                    if need > free * 0.8:
+                        case["status"] = "skipped_vram"
+                        case["logical_elems"] = int(logical)
+                        rows.append(case)
+                        continue
+                    case["logical_elems"] = int(logical)
+                case["contiguous"] = bool(xs[0].is_contiguous())
+                try:
+                    got = op(*xs)
+                    want = ref(*xs)
+                    if v == "overflow_expand":
+                        # bounded comparison: the head plus the high-index
+                        # tail, which is where int32 flat indexing wraps
+                        w = 1 << 20
+                        gf, wf = got.flatten(), want.flatten()
+                        pairs = ((gf[:w], wf[:w]), (gf[-w:], wf[-w:]))
+                    else:
+                        pairs = ((got, want),)
+                    mx = 0
+                    for gp, wp in pairs:
+                        d = ulp_diff(wp, gp) if gp.is_floating_point() \
+                            else (wp != gp).to(torch.int64) * (1 << 62)
+                        if d.numel():
+                            mx = max(mx, int(d.max().item()))
+                        del d
+                    case["status"] = "ok" if mx <= ulp_tol else "mismatch"
+                    case["max_ulp"] = mx
+                    del got, want
+                except Exception as e:  # noqa: BLE001 - report, don't die
+                    case["status"] = "exception"
+                    case["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+                    if not _context_alive(device):
+                        lost = True
+                        case["status"] = "context_killed"
+                rows.append(case)
+    n_bad = sum(1 for r in rows
+                if r["status"] not in ("ok", "skipped_vram",
+                                       "skipped_empty"))
+    return {"cases": rows, "failed": n_bad, "total": len(rows),
+            "context_lost": lost}
+def sweep(factory, grid, iters=30, warmup=5, bytes=None, flops=None,
+          check=None, budget_s=None):
+    """Launch-configuration search: `factory(**point) -> callable`; one
+    bench row per grid point, sorted fastest first. A point whose factory
+    or run raises is reported, not skipped silently. `check(fn) -> bool`
+    is an optional correctness gate recorded per row (a fast wrong config
+    must not win a sweep). `budget_s` caps total sweep time; points not
+    reached are reported as `not_run` rather than silently dropped."""
+    keys = list(grid.keys())
+    def points(i, acc):
+        if i == len(keys):
+            yield dict(acc)
+            return
+        for v in grid[keys[i]]:
+            acc[keys[i]] = v
+            yield from points(i + 1, acc)
+    rows = []
+    t_start = time.perf_counter()
+    over_budget = False
+    for p in points(0, {}):
+        row = dict(p)
+        if over_budget or (budget_s is not None and
+                           time.perf_counter() - t_start > budget_s):
+            over_budget = True
+            row["status"] = "not_run"
+            row["median_ms"] = float("inf")
+            rows.append(row)
+            continue
+        try:
+            fn = factory(**p)
+            if check is not None:
+                row["check"] = bool(check(fn))
+            rep = bench(fn, iters=iters, warmup=warmup, bytes=bytes,
+                        flops=flops)
+            row.update({"median_ms": rep["median_ms"],
+                        "rejected_throttled": rep["rejected_throttled"]})
+            if "achieved_gbs" in rep:
+                row["achieved_gbs"] = rep["achieved_gbs"]
+            if "achieved_tflops" in rep:
+                row["achieved_tflops"] = rep["achieved_tflops"]
+        except Exception as e:  # noqa: BLE001
+            row["error"] = f"{type(e).__name__}: {str(e)[:120]}"
+            row["median_ms"] = float("inf")
+        rows.append(row)
+    rows.sort(key=lambda r: (not r.get("check", True), r["median_ms"]))
+    return rows
+def alloc_stamps(n, device="cuda"):
+    """Device buffer for the MK_STAMP convention (metakernel_cuda/
+    stamps.cuh): the candidate kernel writes clock64() into slot i."""
+    return torch.zeros(n, dtype=torch.int64, device=device)
+def read_stamps(stamps, labels=None, clock_ghz=None):
+    """Decode MK_STAMP phases: consecutive deltas in ticks and in
+    microseconds under the measured (or provided) SM clock. A 1-D buffer
+    (MK_STAMP: block 0 only) yields one delta per phase; a 2-D [B, n]
+    buffer (MK_STAMP_BLOCK) yields min / median / max across blocks per
+    phase, with blocks whose stamps stayed zero excluded."""
+    if clock_ghz is None:
+        clock_ghz = measure_clock(stamps.device)
+    def us(ticks):
+        return round(ticks / clock_ghz / 1000.0, 3)
+    if stamps.dim() == 1:
+        vals = stamps.cpu().tolist()
+        phases = []
+        for i in range(len(vals) - 1):
+            ticks = vals[i + 1] - vals[i]
+            name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+            phases.append({"phase": name, "ticks": ticks, "us": us(ticks)})
+        return {"phases": phases, "clock_ghz": round(clock_ghz, 3)}
+    m = stamps.cpu()
+    live = (m != 0).all(dim=1)
+    m = m[live]
+    deltas = (m[:, 1:] - m[:, :-1]).to(torch.float64)
+    phases = []
+    for i in range(deltas.shape[1]):
+        col = deltas[:, i]
+        name = labels[i] if labels and i < len(labels) else f"phase_{i}"
+        med = float(col.median().item()) if col.numel() else 0.0
+        phases.append({
+            "phase": name,
+            "ticks_median": int(med),
+            "ticks_min": int(col.min().item()) if col.numel() else 0,
+            "ticks_max": int(col.max().item()) if col.numel() else 0,
+            "us_median": us(med),
+            "blocks": int(col.numel()),
+        })
+    return {"phases": phases, "clock_ghz": round(clock_ghz, 3),
+            "blocks_reporting": int(live.sum().item()),
+            "blocks_total": int(live.numel())}

build/torch213-cxx11-cu132-x86_64-linux/_metakernel_cuda_4611d12.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:5c655a858d5a27e42fbfc363c51f5e84f52df5ef7c4be8ad292cc91e4b5f9865
+size 980192

build/torch213-cxx11-cu132-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _metakernel_cuda_4611d12
+ops = torch.ops._metakernel_cuda_4611d12
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_metakernel_cuda_4611d12::{op_name}"

build/torch213-cxx11-cu132-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "metakernel",
+  "id": "_metakernel_cuda_4611d12",
+  "version": 1,
+  "license": "Apache-2.0",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0",
+      "12.0",
+      "8.0",
+      "8.6",
+      "8.9",
+      "9.0"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "cZxKX5I/2EKCMC2gdCwflUsZaQJWJesmAHq2w+fVjXo=",
+      "_metakernel_cuda_4611d12.abi3.so": "XGVahY1aJ+Qvv8NjxR9ehPUt9e98S+itKSzJHktfmGU=",
+      "_ops.py": "uZlR720EcTy8M7BCydmvaZAlKonkwTU3778nhR9yQOE=",
+      "metakernel/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "4611d1213a76a9ec1d12646553a528feb60394ff",
+      "dirty": false
+    }
+  }
+}

build/torch213-cxx11-cu132-x86_64-linux/metakernel/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))