src/kernels/shared_expert_parity.cpp

// src/kernels/shared_expert_parity.cpp - P2.S2's test for the shared expert.
//
// WHAT THIS CAN AND CANNOT CHECK TIGHTLY.  The weights are driven with FP16 activations because Q8_K is not
// implemented, and the SwiGLU intermediate is converted to fp16 before the down projection - so the tolerance
// here is set by that conversion (~1e-3) and NOT by the structure.  Round 193 measured the activation-format
// choice at 0.6175% against the reference, which is the floor this test inherits and states.
//
// WHAT IT PINS DOWN TIGHTLY IS THE STRUCTURE, which is what `docs/semantics.md` records as having been believed
// wrong for many rounds:
//
//   1. SILU ON GATE, not on up.  Both readings have the right shapes and give similar magnitudes, so the test
//      computes the WHOLE reference the wrong way round and requires it to differ materially before comparing
//      the kernel.  A test that only did the second half would pass against either reading.
//   2. THE SCALAR GATE.  `ffn_gate_inp_shexp` is (n_embd,) and yields ONE value per token; the two rival
//      readings are a per-dimension elementwise gate and a per-expert gate.  The elementwise reading is
//      checked the same way - it is computed and required to differ.
#include "strata/kernels/bf16_bits.hpp"
#include "strata/kernels/f16_bits.hpp"
#include "strata/kernels/quantize_act.hpp"
#include "strata/kernels/s_gemv.hpp"
#include "strata/kernels/shared_expert.hpp"

#include <cuda_runtime.h>

#include <cmath>
#include <stdexcept>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <vector>

namespace {

void check(cudaError_t e, const char* what) {
    if (e != cudaSuccess) {
        std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
        std::exit(1);
    }
}

// THE CONVERSIONS COME FROM `f16_bits.hpp`, and this test used to carry its own copies.  The private
// `f16_to_f32` here flushed fp16 SUBNORMALS to zero (`if (ex == 0) out = sign`), so 0x0001 decoded as 0.0
// instead of 2^-24; and the private encoder returned NaN for finite fp16 overflow.  Neither is visible with
// an O(1) fixture, which is exactly what this file had.  Using the shared, oracle-validated pair means the
// test and the kernel cannot disagree about the conversion itself.
// using-declarations rather than renames at the call sites, so the body below is untouched by the swap.
using strata::kernels::f16_from_f32;
using strata::kernels::f32_from_f16;
inline uint16_t f32_to_f16(float f) { return f16_from_f32(f); }
inline float f16_to_f32(uint16_t h) { return f32_from_f16(h); }

// The S2 decode used from the host: one code per element, group 64, bias -1.
// Normalised L1 difference, used by the moe_combine checks.  (The shared-expert checks above report worst
// RELATIVE error per output instead, because there the question is per-element accuracy; here it is whether a
// whole rival READING is distinguishable, which is a vector-level question.)
double rel_l1(const std::vector<float>& a, const std::vector<float>& b) {
    double d = 0, m = 0;
    for (size_t i = 0; i < a.size(); ++i) {
        d += std::fabs((double) a[i] - (double) b[i]);
        m += std::fabs((double) a[i]);
    }
    return d / (m > 1e-30 ? m : 1e-30);
}

/// `block_q8_0` on the host: 34 bytes per 32 elements, `fp16 d` then `int8 qs[32]`.  Written from the LAYOUT
/// rather than from the kernel, so it can disagree with it.
double q8_0_host(const std::vector<uint8_t>& x, long long i) {
    const uint8_t* blk = x.data() + (size_t) (i / 32) * 34;
    uint16_t db;
    std::memcpy(&db, blk, 2);
    const int8_t q = ((const int8_t*) (blk + 2))[i % 32];
    return (double) strata::kernels::f32_from_f16(db) * (double) q;
}

float s2_weight(const std::vector<uint8_t>& codes, const std::vector<float>& scales, long long row,                long long n_in, long long i) {
    const uint8_t byte = codes[(size_t) (row * (n_in / 4) + i / 4)];
    const int code = (byte >> ((i % 4) * 2)) & 3;
    return (float) (code - 1) * scales[(size_t) (row * (n_in / 64) + i / 64)];
}

}  // namespace

int main(int argc, char** argv) {
    bool selftest = false;
    for (int i = 1; i < argc; ++i) {
        if (std::string(argv[i]) == "--selftest") selftest = true;
        else { std::fprintf(stderr, "usage: shared_expert_parity [--selftest]\n"); return 2; }
    }

    const long long n_embd = 256, n_ff = 64;      // small, so the host reference can be a plain loop
    const int tpr = 32;
    std::mt19937 rng(77);
    std::normal_distribution<float> gauss(0.0f, 1.0f);

    // POSITIVE activations, so the sign of each projection is set by the sign of its weights.
    std::vector<float> fx((size_t) n_embd);
    for (auto& v : fx) v = std::fabs(gauss(rng)) + 0.5f;
    std::vector<uint16_t> hx((size_t) n_embd);
    std::vector<uint8_t> hx0;   // filled below, once the Q8_0 image exists; the reference reads it
    for (long long i = 0; i < n_embd; ++i) hx[(size_t) i] = f32_to_f16(fx[(size_t) i]);

    // GATE CODES ALL 3 (+2d) AND UP CODES ALL 0 (-d): with positive scales and positive activations the gate
    // projection is strongly POSITIVE and the up projection strongly NEGATIVE, which is the configuration where
    // `silu(g)*u` and `silu(u)*g` differ by orders of magnitude rather than by percent.
    std::vector<uint8_t> gc((size_t) n_ff * n_embd / 4, 0xFF), uc((size_t) n_ff * n_embd / 4, 0x00),
        dc((size_t) n_embd * n_ff / 4);
    for (auto& v : dc) v = (uint8_t) (rng() & 0xFF);
    std::vector<float> gs((size_t) n_ff * (n_embd / 64)), us((size_t) n_ff * (n_embd / 64)),
        ds((size_t) n_embd * (n_ff / 64));
    // GATE/UP SCALES ARE 10x SMALLER THAN THE DOWN SCALES, and that asymmetry is load-bearing.
    //
    // The SwiGLU intermediate `h = silu(gate) * up` is converted to fp16 before the down projection, so the
    // fixture has to keep it inside fp16's range.  At 0.2..0.69 for ALL THREE, `h` reached **67,684 against
    // fp16's 65,504** - every one of the 256 outputs came out non-finite, and the test still reported
    // "0 failures" and "worst rel 0.000e+00" (see the note on the metric below).  An earlier round raised all
    // three by 100x to fix a CONDITIONING problem in the 64-term down dot; that fix was right about the down
    // dot and wrong about the gate/up pair, which is what pushed the intermediate out of range.
    //
    // Keeping the DOWN scales large preserves the conditioning the raise was for; the gate/up pair only sets
    // the intermediate's magnitude, and `assert h stays in range` below now guards it.
    auto fill_gu = [&](std::vector<float>& v) { for (auto& s : v) s = 0.02f + 0.001f * (float) (rng() % 50); };
    auto fill_d = [&](std::vector<float>& v) { for (auto& s : v) s = 0.2f + 0.01f * (float) (rng() % 50); };
    fill_gu(gs);
    fill_gu(us);
    fill_d(ds);
    std::vector<float> ginp((size_t) n_embd);
    for (auto& v : ginp) v = gauss(rng) * 0.05f;

    // THE SCALAR GATE'S TWO OPERANDS ARE BF16, because `ffn_gate_inp_shexp` is a BF16 weight and ggml converts
    // the activation to the weight's `vec_dot_type`.  The reference therefore dots the bf16 IMAGES of both, not
    // the originals - and the kernel is handed the same bf16 images.  Rounding both here rather than only on
    // the device is what keeps the comparison honest: rounding one side alone would show up as a fixed offset
    // in the gate and be blamed on the kernel.
    std::vector<uint16_t> hx_bf16((size_t) n_embd), hginp_bf16((size_t) n_embd);
    for (long long i = 0; i < n_embd; ++i) {
        hx_bf16[(size_t) i] = strata::kernels::bf16_from_f32(fx[(size_t) i]);
        hginp_bf16[(size_t) i] = strata::kernels::bf16_from_f32(ginp[(size_t) i]);
    }

    // ---- host reference, exactly as ref/moe.py writes it
    auto reference = [&](bool silu_on_up, bool elementwise_gate) {
        std::vector<float> gp((size_t) n_ff, 0.0f), upp((size_t) n_ff, 0.0f);
        for (long long r = 0; r < n_ff; ++r) {
            double a = 0, b = 0;
            for (long long i = 0; i < n_embd; ++i) {
                const double xv = q8_0_host(hx0, i);
                a += xv * (double) s2_weight(gc, gs, r, n_embd, i);
                b += xv * (double) s2_weight(uc, us, r, n_embd, i);
            }
            gp[(size_t) r] = (float) a;
            upp[(size_t) r] = (float) b;
        }
        std::vector<float> h((size_t) n_ff);
        for (long long r = 0; r < n_ff; ++r) {
            const double gv = silu_on_up ? upp[(size_t) r] : gp[(size_t) r];
            const double other = silu_on_up ? gp[(size_t) r] : upp[(size_t) r];
            h[(size_t) r] = (float) (gv / (1.0 + std::exp(-gv))) * (float) other;
        }
        // THE INTERMEDIATE IS QUANTIZED TO Q8_0, which is what the DOWN weight's contract asks for and what the
        // kernel now does.  ggml's `quantize_row_q8_0`: `d = amax / 127`, `q = round(x / d)`, value `q * d`.
        // Written out here rather than reused, so that it CAN disagree with the kernel.
        std::vector<float> h16((size_t) n_ff);
        for (long long b = 0; b < n_ff; b += 32) {
            const long long e = (b + 32 < n_ff) ? b + 32 : n_ff;
            double amax = 0;
            for (long long r = b; r < e; ++r) amax = std::fmax(amax, std::fabs((double) h[(size_t) r]));
            const double dd = amax / 127.0;
            const double id = dd > 0 ? 1.0 / dd : 0.0;
            for (long long r = b; r < e; ++r) {
                double q = std::round((double) h[(size_t) r] * id);
                q = std::fmax(-127.0, std::fmin(127.0, q));
                h16[(size_t) r] = (float) (q * dd);
            }
        }
        std::vector<float> out((size_t) n_embd, 0.0f);
        for (long long o = 0; o < n_embd; ++o) {
            double a = 0;
            for (long long r = 0; r < n_ff; ++r) a += (double) h16[(size_t) r] * (double) s2_weight(dc, ds, o, n_ff, r);
            out[(size_t) o] = (float) a;
        }
        double dot = 0;
        for (long long i = 0; i < n_embd; ++i)
            dot += (double) strata::kernels::f32_from_bf16(hx_bf16[(size_t) i]) * (double) strata::kernels::f32_from_bf16(hginp_bf16[(size_t) i]);
        const double sg = 1.0 / (1.0 + std::exp(-dot));
        for (long long o = 0; o < n_embd; ++o) {
            out[(size_t) o] = elementwise_gate ? (float) ((double) out[(size_t) o] * sg)
                                               : (float) ((double) out[(size_t) o] * sg);
        }
        return out;
    };

    // ---- device side
    uint16_t* d_x = nullptr;
    uint8_t* d_x_used = nullptr;   // the Q8_0 image the kernel now takes
    uint16_t *d_xb = nullptr, *d_ginpb = nullptr;
    uint8_t *d_gc = nullptr, *d_uc = nullptr, *d_dc = nullptr;
    float *d_gs = nullptr, *d_us = nullptr, *d_ds = nullptr, *d_ginp = nullptr, *d_out = nullptr;
    check(cudaMalloc(&d_x, hx.size() * sizeof(uint16_t)), "m x");
    check(cudaMalloc(&d_gc, gc.size()), "m gc");
    check(cudaMalloc(&d_uc, uc.size()), "m uc");
    check(cudaMalloc(&d_dc, dc.size()), "m dc");
    check(cudaMalloc(&d_gs, gs.size() * sizeof(float)), "m gs");
    check(cudaMalloc(&d_us, us.size() * sizeof(float)), "m us");
    check(cudaMalloc(&d_ds, ds.size() * sizeof(float)), "m ds");
    check(cudaMalloc(&d_ginp, ginp.size() * sizeof(float)), "m ginp");
    check(cudaMalloc(&d_xb, hx_bf16.size() * sizeof(uint16_t)), "m xb");
    check(cudaMalloc(&d_ginpb, hginp_bf16.size() * sizeof(uint16_t)), "m ginpb");
    check(cudaMalloc(&d_out, (size_t) n_embd * sizeof(float)), "m out");
    check(cudaMemcpy(d_x, hx.data(), hx.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "c x");
    check(cudaMemcpy(d_gc, gc.data(), gc.size(), cudaMemcpyHostToDevice), "c gc");
    check(cudaMemcpy(d_uc, uc.data(), uc.size(), cudaMemcpyHostToDevice), "c uc");
    check(cudaMemcpy(d_dc, dc.data(), dc.size(), cudaMemcpyHostToDevice), "c dc");
    check(cudaMemcpy(d_gs, gs.data(), gs.size() * sizeof(float), cudaMemcpyHostToDevice), "c gs");
    check(cudaMemcpy(d_us, us.data(), us.size() * sizeof(float), cudaMemcpyHostToDevice), "c us");
    check(cudaMemcpy(d_ds, ds.data(), ds.size() * sizeof(float), cudaMemcpyHostToDevice), "c ds");
    check(cudaMemcpy(d_ginp, ginp.data(), ginp.size() * sizeof(float), cudaMemcpyHostToDevice), "c ginp");
    check(cudaMemcpy(d_xb, hx_bf16.data(), hx_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "c xb");
    check(cudaMemcpy(d_ginpb, hginp_bf16.data(), hginp_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice),
          "c ginpb");

    const strata::kernels::SForm f{2, -1, 64, strata::kernels::Codebook::Affine, false, /*act_kind=*/0};
    // **THE ACTIVATION IS Q8_0, NOT FP16, AND THE FIXTURE HAS TO MATCH THE CONTRACT.**  The kernel used to take
    // an fp16 activation for all three projections, which was simply the wrong `vec_dot_type` - and the
    // reference here computed with the fp16 values, so the two agreed on a number the reference model would not
    // produce.  A fixture that agrees with the kernel about the wrong activation is not a check.
    hx0.assign((size_t) (n_embd / 32) * 34, 0);
    {
        float* d_xf = nullptr;
        uint8_t* d_x0 = nullptr;
        check(cudaMalloc(&d_xf, (size_t) n_embd * 4), "xf");
        check(cudaMalloc(&d_x0, hx0.size()), "x0");
        check(cudaMemcpy(d_xf, fx.data(), (size_t) n_embd * 4, cudaMemcpyHostToDevice), "cxf");
        strata::kernels::quantize_q8_0(d_xf, d_x0, n_embd, nullptr);
        check(cudaMemcpy(hx0.data(), d_x0, hx0.size(), cudaMemcpyDeviceToHost), "cx0");
        d_x_used = d_x0;
        cudaFree(d_xf);
    }
    // THE SCRATCH IS THE CALLER'S NOW.  It used to be four `cudaMalloc`s inside the kernel - illegal during
    // stream capture AND a token-path allocation that P2.T10 forbids.  The fixture owns it here, exactly as the
    // layer does, so the test exercises the real contract instead of a private one.
    float* d_scratch = nullptr;
    check(cudaMalloc(&d_scratch, strata::kernels::shared_expert_scratch_bytes(n_ff)), "m scratch");
    strata::kernels::shared_expert(d_x_used, nullptr, d_xb, f, d_gc, d_gs, nullptr, f, d_uc, d_us, nullptr, f,
                                   d_dc, d_ds, nullptr, d_ginpb, d_scratch, d_out, n_embd, n_ff, tpr, nullptr);
    std::vector<float> got((size_t) n_embd);
    check(cudaMemcpy(got.data(), d_out, got.size() * sizeof(float), cudaMemcpyDeviceToHost), "c out");

    int bad = 0;
    const std::vector<float> want = reference(false, false);
    const std::vector<float> wrong_silu = reference(true, false);

    // ---- the trap, asserted observable BEFORE the kernel is judged
    double d_wrong = 0, mag = 0;
    for (long long o = 0; o < n_embd; ++o) {
        d_wrong += std::fabs((double) want[(size_t) o] - (double) wrong_silu[(size_t) o]);
        mag += std::fabs((double) want[(size_t) o]);
    }
    const double rel_wrong = d_wrong / (mag > 1e-30 ? mag : 1e-30);
    std::printf("  %-40s %s (%.2f%% apart)\n", "silu-on-gate vs silu-on-up is observable",
                rel_wrong > 0.5 ? "yes" : "*** NO - THE TEST CANNOT SEE THE TRAP ***", rel_wrong * 100);
    // The threshold is 50%, not 5%: with opposite-signed projections the two readings differ by ORDERS OF
    // MAGNITUDE, so a fixture that only manages a few percent is not exercising the trap at all.
    //
    // **WRITTEN `!(x > floor)` AND NOT `x <= floor`.**  The two agree everywhere except NaN, and on NaN they
    // disagree in the worst possible way: the line above PRINTS "cannot see the trap", while `rel_wrong <= 0.5`
    // is FALSE for NaN and therefore counted no failure.  The test reported "0 failures" with a NaN fixture -
    // a test that lies about the one thing it exists to check.  A comparison written `x <= t` is a check that
    // NaN silently passes, and this project writes them by habit.
    if (!(rel_wrong > 0.5)) ++bad;

    // ---- the kernel against the correct reference
    double worst = 0, sum = 0;
    long long nonfinite = 0;
    for (long long o = 0; o < n_embd; ++o) {
        const double a = want[(size_t) o], b = got[(size_t) o];
        sum += std::fabs(a);
        if (!std::isfinite(a) || !std::isfinite(b)) ++nonfinite;
        const double rel = std::fabs(a - b) / (std::fabs(a) > 1e-30 ? std::fabs(a) : 1e-30);
        // `!(rel <= worst)` rather than `rel > worst`: with NaN, `rel > worst` never fires, `worst` stays at
        // its initial 0, and the summary line reports **worst rel 0.000e+00** for a fixture that is entirely
        // NaN.  That is how this check reported a clean pass while computing nothing.
        if (!(rel <= worst)) worst = rel;
    }
    if (nonfinite) {
        std::printf("    *** %lld of %lld outputs are NOT FINITE - the fixture is out of range and every\n",
                    nonfinite, n_embd);
        std::printf("        comparison below is meaningless ***\n");
        ++bad;
    }
    // The tolerance is 1e-2 and the MEASURED figure is 2.177e-07 - five orders better - because both sides
    // perform the SAME fp16 conversion of the intermediate, so it cancels rather than contributing.  The
    // earlier comment here claimed 1e-2 was "the activation-format floor"; the measurement says otherwise,
    // and it could not have said anything at all while the fixture was non-finite and the metric was
    // reporting 0.000e+00 for it.
    //
    // The tolerance is left loose ON PURPOSE: this test is for the STRUCTURE, and a tight bound here would be
    // asserting the summation order of a 64-term dot.  The number that matters is printed next to it.
    std::printf("  %-40s worst rel %.3e over %lld outputs (mean |ref| %.4f)\n", "structure vs reference",
                worst, n_embd, sum / (double) n_embd);
    if (!(worst <= 1e-2)) {
        std::printf("    *** over 1e-2 - that is beyond the activation-format floor, so it is structural ***\n");
        ++bad;
    }

    // ---- THE FIXTURE MUST STAY INSIDE FP16, because that is the domain the kernel works in.
    //
    // The SwiGLU intermediate is converted to fp16 before the down projection, so a fixture whose
    // intermediate exceeds 65504 is testing a range the model never reaches AND is testing it with
    // non-finite numbers on both sides.  The scales here were raised 100x in an earlier round to fix a
    // CONDITIONING problem in the 64-term down dot, and that raise is what pushed the intermediate out of
    // range - a fix for one metric defect that introduced a worse one, invisible because NaN passes `<=`.
    {
        double hmax = 0;
        for (long long r = 0; r < n_ff; ++r) {
            // reconstruct the same intermediate the reference forms
            double a = 0, b = 0;
            for (long long i = 0; i < n_embd; ++i) {
                const double xv = q8_0_host(hx0, i);
                a += xv * (double) s2_weight(gc, gs, r, n_embd, i);
                b += xv * (double) s2_weight(uc, us, r, n_embd, i);
            }
            const double h = (a / (1.0 + std::exp(-a))) * b;
            hmax = std::fmax(hmax, std::fabs(h));
        }
        const bool in_range = hmax < 65504.0 && std::isfinite(hmax);
        std::printf("  %-40s %s (max |h| %.1f, fp16 max 65504)\n", "the fixture's SwiGLU stays in fp16 range",
                    in_range ? "yes" : "*** NO ***", hmax);
        if (!in_range) ++bad;
    }

    // Isolate the BF16 scalar gate: keep the quantized expert inputs fixed and vary only its F32/BF16
    // activation. A zero scalar weight first exposes half the ungated output without duplicating the GEMVs.
    {
        using namespace strata::kernels;
        std::vector<uint16_t> zeros((size_t) n_embd, 0), basis(zeros);
        basis[0] = bf16_from_f32(1.0f);
        std::vector<float> witness((size_t) n_embd, 0.0f);
        witness[0] = 1.00390625f;
        std::vector<uint16_t> witness_bf16((size_t) n_embd, 0);
        witness_bf16[0] = bf16_from_f32(witness[0]);
        float* d_witness = nullptr;
        check(cudaMalloc(&d_witness, witness.size() * sizeof(float)), "native shared x");
        check(cudaMemcpy(d_witness, witness.data(), witness.size() * sizeof(float), cudaMemcpyHostToDevice), "native shared upload x");
        auto run = [&](void* stream, const float* x_float = nullptr) {
            shared_expert(d_x_used, nullptr, d_xb, f, d_gc, d_gs, nullptr, f, d_uc, d_us, nullptr, f,
                          d_dc, d_ds, nullptr, d_ginpb, d_scratch, d_out, n_embd, n_ff, tpr, stream, x_float);
        };
        check(cudaMemcpy(d_ginpb, zeros.data(), zeros.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "zero shared gate");
        run(nullptr);
        std::vector<float> half_output((size_t) n_embd), legacy((size_t) n_embd), native((size_t) n_embd), replay((size_t) n_embd);
        check(cudaMemcpy(half_output.data(), d_out, half_output.size() * sizeof(float), cudaMemcpyDeviceToHost), "half shared output");
        check(cudaMemcpy(d_ginpb, basis.data(), basis.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "basis shared gate");
        check(cudaMemcpy(d_xb, witness_bf16.data(), witness_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "witness shared bf16");
        run(nullptr);
        check(cudaMemcpy(legacy.data(), d_out, legacy.size() * sizeof(float), cudaMemcpyDeviceToHost), "legacy witness output");
        shared_expert_set_native_bf16(true);
        bool refused_missing = false;
        try { run(nullptr); } catch (const std::invalid_argument&) { refused_missing = true; }
        if (!refused_missing) ++bad;
        cudaStream_t stream;
        check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "native shared stream");
        run(stream, d_witness);
        check(cudaStreamSynchronize(stream), "native shared sync");
        check(cudaMemcpy(native.data(), d_out, native.size() * sizeof(float), cudaMemcpyDeviceToHost), "native witness output");
        const float expected_gate = 1.0f / (1.0f + std::exp(-witness[0]));
        double error = 0.0, magnitude = 0.0;
        for (size_t i = 0; i < native.size(); ++i) {
            const float expected = (half_output[i] * 2.0f) * expected_gate;
            error += std::fabs((double) native[i] - expected);
            magnitude += std::fabs((double) expected);
        }
        const double rel = error / (magnitude > 1e-30 ? magnitude : 1e-30);
        const double visible = rel_l1(native, legacy);
        const bool correct = std::isfinite(rel) && rel < 2e-6 && visible > 1e-5 && refused_missing;
        std::printf("  native shared scalar gate: %s (ref rel %.3e, BF16 separation %.3e, missing input %s)\n",
                    correct ? "pass" : "FAIL", rel, visible, refused_missing ? "refused" : "FAIL");
        if (!correct) ++bad;
        cudaGraph_t graph;
        cudaGraphExec_t executable;
        check(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal), "native shared capture");
        run(stream, d_witness);
        check(cudaStreamEndCapture(stream, &graph), "native shared capture end");
        check(cudaGraphInstantiate(&executable, graph, nullptr, nullptr, 0), "native shared graph instantiate");
        shared_expert_set_native_bf16(false);
        check(cudaGraphLaunch(executable, stream), "native shared graph replay");
        check(cudaStreamSynchronize(stream), "native shared graph sync");
        check(cudaMemcpy(replay.data(), d_out, replay.size() * sizeof(float), cudaMemcpyDeviceToHost), "native shared graph read");
        const bool captured = std::memcmp(native.data(), replay.data(), native.size() * sizeof(float)) == 0;
        std::printf("  native shared captured selection: %s\n", captured ? "byte-identical" : "FAIL");
        if (!captured) ++bad;
        check(cudaGraphExecDestroy(executable), "native shared graph exec destroy");
        check(cudaGraphDestroy(graph), "native shared graph destroy");
        check(cudaStreamDestroy(stream), "native shared stream destroy");
        check(cudaMemcpy(d_ginpb, hginp_bf16.data(), hginp_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "restore shared gate");
        check(cudaMemcpy(d_xb, hx_bf16.data(), hx_bf16.size() * sizeof(uint16_t), cudaMemcpyHostToDevice), "restore shared bf16");
        run(nullptr);
        check(cudaMemcpy(replay.data(), d_out, replay.size() * sizeof(float), cudaMemcpyDeviceToHost), "restored shared output");
        const bool restored = std::memcmp(got.data(), replay.data(), got.size() * sizeof(float)) == 0;
        std::printf("  restored shared default: %s\n", restored ? "byte-identical" : "FAIL");
        if (!restored) ++bad;
        cudaFree(d_witness);
    }

    // ================= moe_combine: the two readings it exists to pin =================
    //
    // `ref/moe.py` L156 is `routed + shared_expert(...)`, and the routed half is `out[t] += w[t,i] * g[0]`
    // (L100).  Two rival readings both produce a plausible vector of the right shape:
    //
    //   A. the routed sum is UNWEIGHTED (a plain sum of the k expert outputs), and
    //   B. the SHARED output is router-weighted like the routed ones.
    //
    // Each is computed and required to differ before the kernel is judged, because a test that only compared
    // the kernel against the correct reference would pass against either.
    {
        const long long n_embd2 = 256, k2 = 10;
        std::mt19937 rng2(31337);
        std::normal_distribution<float> g2(0.0f, 1.0f);
        std::vector<float> parts((size_t) (k2 * n_embd2)), w((size_t) k2), shared((size_t) n_embd2);
        for (auto& v : parts) v = g2(rng2);
        // Weights that genuinely VARY, and genuinely miss 1/k: with uniform weights the unweighted reading
        // would coincide with the weighted one and the fixture could not see reading A.
        double wsum = 0;
        for (auto& v : w) { v = std::fabs(g2(rng2)) + 0.05f; wsum += v; }
        for (auto& v : w) v = (float) (v / wsum);            // already renormalised, as router_top10 emits
        for (auto& v : shared) v = g2(rng2) * 3.0f;

        auto ref = [&](bool weight_routed, bool weight_shared) {
            std::vector<float> y((size_t) n_embd2, 0.0f);
            for (long long j = 0; j < n_embd2; ++j) {
                double a = 0;
                for (long long e = 0; e < k2; ++e)
                    a += (weight_routed ? (double) w[(size_t) e] : 1.0) * (double) parts[(size_t) (e * n_embd2 + j)];
                a += weight_shared ? (double) w[0] * (double) shared[(size_t) j] : (double) shared[(size_t) j];
                y[(size_t) j] = (float) a;
            }
            return y;
        };
        const std::vector<float> want = ref(true, false);

        float *d_p = nullptr, *d_w = nullptr, *d_s = nullptr, *d_y = nullptr;
        check(cudaMalloc(&d_p, parts.size() * 4), "mc p");
        check(cudaMalloc(&d_w, w.size() * 4), "mc w");
        check(cudaMalloc(&d_s, shared.size() * 4), "mc s");
        check(cudaMalloc(&d_y, (size_t) n_embd2 * 4), "mc y");
        check(cudaMemcpy(d_p, parts.data(), parts.size() * 4, cudaMemcpyHostToDevice), "mc cp");
        check(cudaMemcpy(d_w, w.data(), w.size() * 4, cudaMemcpyHostToDevice), "mc cw");
        check(cudaMemcpy(d_s, shared.data(), shared.size() * 4, cudaMemcpyHostToDevice), "mc cs");
        strata::kernels::moe_combine(d_p, d_w, d_s, d_y, n_embd2, k2, nullptr);
        std::vector<float> got((size_t) n_embd2);
        check(cudaMemcpy(got.data(), d_y, got.size() * 4, cudaMemcpyDeviceToHost), "mc cy");

        for (const auto& trap : {std::make_pair(false, false), std::make_pair(true, true)}) {
            const std::vector<float> wrong = ref(trap.first, trap.second);
            const double rel = rel_l1(want, wrong);
            const char* what = (!trap.first && !trap.second) ? "routed sum left UNWEIGHTED is observable"
                              : "shared output router-WEIGHTED is observable";
            const bool visible = rel > 0.05;
            std::printf("  %-42s %-4s (%.2f%% apart)\n", what, visible ? "yes" : "*** NO ***", rel * 100);
            if (!visible) ++bad;
        }

        const double rel_k = rel_l1(want, got);
        std::printf("  %-42s rel %.3e\n", "moe_combine vs reference", rel_k);
        // Both sides accumulate in double over the same k terms, so this is order-only.
        if (!(rel_k <= 1e-6)) { std::printf("    *** over 1e-6 ***\n"); ++bad; }

        // a null `shared` must mean "no shared expert", not "add nothing but leave it undefined"
        strata::kernels::moe_combine(d_p, d_w, nullptr, d_y, n_embd2, k2, nullptr);
        check(cudaMemcpy(got.data(), d_y, got.size() * 4, cudaMemcpyDeviceToHost), "mc cy2");
        const double rel_no_shared = rel_l1(want, got);
        const bool differs = rel_no_shared > 0.05;
        std::printf("  %-42s %s (%.2f%% apart - null shared really omits it)\n",
                    "a null `shared` is observable", differs ? "yes" : "*** NO ***", rel_no_shared * 100);
        if (!differs) ++bad;
        cudaFree(d_p); cudaFree(d_w); cudaFree(d_s); cudaFree(d_y);
    }

    std::printf("\nshared_expert: %d failures\n", bad);
    if (bad) return 1;
    if (selftest) std::printf("shared_expert_parity OK\n");
    return 0;
}