File size: 21,922 Bytes
bbb6388 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 | // src/kernels/gdn_parity.cpp - P2.S2's test for the GDN recurrence, the conv and the two norms.
//
// `ref/gdn.py` carries eleven PROPERTY checks, several of which exist because a rival reading of the SOURCE
// produces well-formed output. The four that the kernels here can be wrong about are re-pinned below, each
// asserted OBSERVABLE against a deliberately-wrong computation before the kernel is judged:
//
// 1. MODULO head pairing, h % h_k, not INTERLEAVE. With H_k=2 and H_v=4 the two give [0,1,0,1] and
// [0,0,1,1]: they differ at h=1 and h=2. Both produce a full-rank state of the right shape.
// 2. Decay applied BEFORE the rank-1 update. Applying it after is a one-line move that keeps every shape
// and every magnitude.
// 3. The conv's SLIDE DIRECTION. `kernel[0]` must read the OLDEST state row and the new input must land in
// the LAST state row; reversing either is invisible on a symmetric state, so the state rows here are
// labelled so each position is identifiable.
// 4. `l2_norm`'s `+ eps` is an absolute floor on the SQUARED NORM, not on the mean. The mean reading
// differs by sqrt(S) = 11.3x at S = 128. `ref/gdn.py` PROPERTY 1 instruments the derivation, so it is
// not taken on trust there either.
//
// It also checks the STATE LAYOUT, which is the one place these kernels intentionally differ from the
// reference: (S, h_v, S) with j fastest instead of (S, S, h_v). A layout mix-up is silent, so the state is
// filled with a value that encodes its own coordinates.
#include "strata/kernels/gdn.hpp"
#include <cuda_runtime.h>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <random>
#include <string>
#include <vector>
namespace {
void check(cudaError_t e, const char* what) {
if (e != cudaSuccess) {
std::fprintf(stderr, "%s: %s\n", what, cudaGetErrorString(e));
std::exit(1);
}
}
float sigmoid_f(float x) { return 1.0f / (1.0f + std::exp(-x)); }
/// The reference's `gdn_step`, with switches for the rival readings.
///
/// `state` is passed in the REFERENCE layout (S, S, h_v) - i.e. (i, j, h) - so that this function is a
/// transcription of `ref/gdn.py` and not of the kernel.
void ref_step(std::vector<double>& state, const std::vector<float>& q, const std::vector<float>& k,
const std::vector<float>& v, const std::vector<float>& gate, const std::vector<float>& beta,
int S, int h_k, int h_v, std::vector<float>& o, bool modulo_heads, bool decay_first) {
auto at = [&](int i, int j, int h) -> double& { return state[(size_t) (i * S + j) * h_v + h]; };
std::vector<float> oo((size_t) h_v * S, 0.0f);
for (int h = 0; h < h_v; ++h) {
const int src = modulo_heads ? (h % h_k) : (h / (h_v / h_k));
const double dec = std::exp((double) gate[(size_t) h]);
const double b = (double) beta[(size_t) h];
for (int j = 0; j < S; ++j) {
if (decay_first) for (int i = 0; i < S; ++i) at(i, j, h) *= dec;
double sk = 0;
for (int i = 0; i < S; ++i) sk += at(i, j, h) * (double) k[(size_t) (src * S + i)];
const double d = ((double) v[(size_t) (h * S + j)] - sk) * b;
for (int i = 0; i < S; ++i) at(i, j, h) += (double) k[(size_t) (src * S + i)] * d;
if (!decay_first) for (int i = 0; i < S; ++i) at(i, j, h) *= dec;
double dot = 0;
for (int i = 0; i < S; ++i) dot += at(i, j, h) * (double) q[(size_t) (src * S + i)];
oo[(size_t) (h * S + j)] = (float) dot;
}
}
o = oo;
}
/// `ref/gdn.py::conv_step` in its own (d_conv, C) row-major orientation.
void ref_conv(std::vector<float>& cs, const std::vector<float>& x, const std::vector<float>& kern, int C,
int dc, std::vector<float>& out) {
out.assign((size_t) C, 0.0f);
for (int c = 0; c < C; ++c) {
double acc = 0;
for (int i = 0; i < dc - 1; ++i) acc += (double) cs[(size_t) (i * C + c)] * (double) kern[(size_t) (i * C + c)];
acc += (double) x[(size_t) c] * (double) kern[(size_t) ((dc - 1) * C + c)];
out[(size_t) c] = (float) acc;
for (int i = 0; i < dc - 2; ++i) cs[(size_t) (i * C + c)] = cs[(size_t) ((i + 1) * C + c)];
cs[(size_t) ((dc - 2) * C + c)] = x[(size_t) c];
}
}
double rel_l1(const std::vector<float>& a, const std::vector<float>& b) {
double d = 0, m = 0;
for (size_t i = 0; i < a.size(); ++i) {
d += std::fabs((double) a[i] - (double) b[i]);
m += std::fabs((double) a[i]);
}
return d / (m > 1e-30 ? m : 1e-30);
}
} // namespace
int main(int argc, char** argv) {
bool selftest = false;
for (int i = 1; i < argc; ++i) {
if (std::string(argv[i]) == "--selftest") selftest = true;
else { std::fprintf(stderr, "usage: gdn_parity [--selftest]\n"); return 2; }
}
const int S = 128, h_k = 16, h_v = 48; // the artifact's real geometry
std::mt19937 rng(2024);
std::normal_distribution<float> gauss(0.0f, 1.0f);
int bad = 0;
// ================= 1. the recurrence =================
{
// The state encodes its own coordinates, so a layout mix-up cannot pass: reading (i,j,h) where the
// buffer holds (i,h,j) gives a DIFFERENT number, not a wrong-but-plausible one.
std::vector<double> st_ref((size_t) S * S * h_v);
std::vector<float> st_dev((size_t) S * h_v * S);
for (int i = 0; i < S; ++i)
for (int j = 0; j < S; ++j)
for (int h = 0; h < h_v; ++h) {
const float val = (float) (0.001 * (i + 2 * j + 3 * h) - 1.0);
st_ref[(size_t) (i * S + j) * h_v + h] = val;
st_dev[(size_t) (i * h_v + h) * S + j] = val; // (S, h_v, S), j fastest
}
std::vector<float> q((size_t) h_k * S), k((size_t) h_k * S), v((size_t) h_v * S), gate((size_t) h_v),
beta((size_t) h_v);
for (auto& x : q) x = gauss(rng) * 0.1f;
for (auto& x : k) x = gauss(rng) * 0.1f;
for (auto& x : v) x = gauss(rng);
for (auto& x : gate) x = -(2.0f + 4.0f * (float) (rng() % 100) / 100.0f); // dec in (0.0025, 0.135]
for (auto& x : beta) x = (float) (rng() % 100) / 100.0f;
std::vector<float> want_o;
std::vector<double> st_mod = st_ref;
ref_step(st_mod, q, k, v, gate, beta, S, h_k, h_v, want_o, true, true);
// the traps
{
std::vector<double> a = st_ref;
std::vector<float> oo;
ref_step(a, q, k, v, gate, beta, S, h_k, h_v, oo, false, true);
const double rel = rel_l1(want_o, oo);
std::printf(" %-42s %-4s (%.2f%% apart)\n", "modulo vs interleave head pairing",
rel > 0.05 ? "yes" : "*** NO ***", rel * 100);
if (!(rel > 0.05)) ++bad;
}
{
std::vector<double> a = st_ref;
std::vector<float> oo;
ref_step(a, q, k, v, gate, beta, S, h_k, h_v, oo, true, false);
const double rel = rel_l1(want_o, oo);
std::printf(" %-42s %-4s (%.2f%% apart)\n", "decay before vs after the update",
rel > 0.05 ? "yes" : "*** NO ***", rel * 100);
// The two readings differ by exactly `k (x) (beta * v * (1 - dec))`, so the fixture is only able
// to see the trap when `dec` is far from 1 and `beta` is far from 0. The first version used
// gate in [-0.99, 0] - dec in [0.37, 1] - and managed 3.25%, which is a threshold the wrong
// reading nearly clears. Deep decay is also the realistic regime: `ssm_a = -exp(A_log) < 0` and
// softplus is positive, so gate is comfortably negative in the real model.
if (!(rel > 0.05)) ++bad;
}
// the kernel
float *d_st = nullptr, *d_q = nullptr, *d_k = nullptr, *d_v = nullptr, *d_g = nullptr, *d_b = nullptr,
*d_o = nullptr;
check(cudaMalloc(&d_st, st_dev.size() * 4), "st");
check(cudaMalloc(&d_q, q.size() * 4), "q");
check(cudaMalloc(&d_k, k.size() * 4), "k");
check(cudaMalloc(&d_v, v.size() * 4), "v");
check(cudaMalloc(&d_g, gate.size() * 4), "g");
check(cudaMalloc(&d_b, beta.size() * 4), "b");
check(cudaMalloc(&d_o, (size_t) h_v * S * 4), "o");
check(cudaMemcpy(d_st, st_dev.data(), st_dev.size() * 4, cudaMemcpyHostToDevice), "cst");
check(cudaMemcpy(d_q, q.data(), q.size() * 4, cudaMemcpyHostToDevice), "cq");
check(cudaMemcpy(d_k, k.data(), k.size() * 4, cudaMemcpyHostToDevice), "ck");
check(cudaMemcpy(d_v, v.data(), v.size() * 4, cudaMemcpyHostToDevice), "cv");
check(cudaMemcpy(d_g, gate.data(), gate.size() * 4, cudaMemcpyHostToDevice), "cg");
check(cudaMemcpy(d_b, beta.data(), beta.size() * 4, cudaMemcpyHostToDevice), "cb");
strata::kernels::GdnShapes sh{S, h_k, h_v};
strata::kernels::gdn_step(d_st, d_q, d_k, d_v, d_g, d_b, d_o, sh, nullptr);
std::vector<float> got_o((size_t) h_v * S), got_st(st_dev.size());
check(cudaMemcpy(got_o.data(), d_o, got_o.size() * 4, cudaMemcpyDeviceToHost), "cgo");
check(cudaMemcpy(got_st.data(), d_st, got_st.size() * 4, cudaMemcpyDeviceToHost), "cgst");
// o, and the STATE read back through the declared layout
const double rel_o = rel_l1(want_o, got_o);
double rel_st = 0, mag_st = 0;
for (int i = 0; i < S; ++i)
for (int j = 0; j < S; ++j)
for (int h = 0; h < h_v; ++h) {
const size_t dev = (size_t) (i * h_v + h) * S + j;
const double w = st_mod[(size_t) (i * S + j) * h_v + h];
rel_st += std::fabs(w - (double) got_st[dev]);
mag_st += std::fabs(w);
}
rel_st /= (mag_st > 1e-30 ? mag_st : 1e-30);
std::printf("\n %-42s rel %.3e\n", "o vs reference", rel_o);
std::printf(" %-42s rel %.3e\n", "state (read via (S,h_v,S)) vs reference", rel_st);
// The kernel accumulates in f32 and the reference in f64, and the state is a RECURRENCE over 128
// terms, so 1e-5 is the arithmetic's floor here and not a structural allowance.
if (!(rel_o <= 1e-5)) { std::printf(" *** o over 1e-5 ***\n"); ++bad; }
if (!(rel_st <= 1e-5)) { std::printf(" *** state over 1e-5 ***\n"); ++bad; }
// a zero state stays zero, and a zero gate/beta leaves the state untouched (the reference's
// PROPERTY 4 setup) - a cheap independent invariant that no layout can fake
{
std::vector<float> zero(st_dev.size(), 0.0f), zg((size_t) h_v, 0.0f), zb((size_t) h_v, 0.0f);
check(cudaMemcpy(d_st, zero.data(), zero.size() * 4, cudaMemcpyHostToDevice), "z0");
check(cudaMemcpy(d_g, zg.data(), zg.size() * 4, cudaMemcpyHostToDevice), "z1");
check(cudaMemcpy(d_b, zb.data(), zb.size() * 4, cudaMemcpyHostToDevice), "z2");
strata::kernels::gdn_step(d_st, d_q, d_k, d_v, d_g, d_b, d_o, sh, nullptr);
std::vector<float> zz(zero.size());
check(cudaMemcpy(zz.data(), d_st, zz.size() * 4, cudaMemcpyDeviceToHost), "z3");
double nz = 0;
for (float x : zz) nz += std::fabs(x);
const bool ok = nz == 0.0;
std::printf(" %-42s %s\n", "gate=beta=0 from a zero state is a no-op", ok ? "yes" : "*** NO ***");
if (!ok) ++bad;
}
// ephemeral state: unchanged when the state is untouched but beta=1 — the delta rule writing k*d
cudaFree(d_st); cudaFree(d_q); cudaFree(d_k); cudaFree(d_v); cudaFree(d_g); cudaFree(d_b); cudaFree(d_o);
}
// ================= 2. the conv =================
{
const int C = 24, dc = 4; // small, so the host reference is a plain loop
// TWO DIFFERENT LAYOUTS MEET HERE, and this is where the first version of this test failed at 0.136.
// `ref/gdn.py` carries the conv state as (d_conv-1, C) ROW-MAJOR - (i, c) at i*C + c - because it is
// numpy. The kernel carries it as (C, d_conv-1) with c fastest - c*(dc-1) + i - because that is
// ggml's `ne = {d_conv-1, C}` with ne0 fast, and it is a runtime buffer so either is legal. The
// mapping between them is written out below rather than assumed, in both directions.
std::vector<float> cs_ref((size_t) (dc - 1) * C), x((size_t) C), kern((size_t) dc * C);
for (int i = 0; i < dc - 1; ++i)
for (int c = 0; c < C; ++c) cs_ref[(size_t) (i * C + c)] = (float) (10 * (i + 1) + c); // labelled
for (int c = 0; c < C; ++c) x[(size_t) c] = (float) (100 + c);
for (size_t i = 0; i < kern.size(); ++i) kern[i] = gauss(rng);
auto to_dev_layout = [&](const std::vector<float>& r) {
std::vector<float> d(r.size());
for (int i = 0; i < dc - 1; ++i)
for (int c = 0; c < C; ++c) d[(size_t) (c * (dc - 1) + i)] = r[(size_t) (i * C + c)];
return d;
};
auto to_ref_layout = [&](const std::vector<float>& d) {
std::vector<float> r(d.size());
for (int i = 0; i < dc - 1; ++i)
for (int c = 0; c < C; ++c) r[(size_t) (i * C + c)] = d[(size_t) (c * (dc - 1) + i)];
return r;
};
std::vector<float> cs = to_dev_layout(cs_ref);
std::vector<float> want;
std::vector<float> cs_want_ref = cs_ref;
ref_conv(cs_want_ref, x, kern, C, dc, want);
const std::vector<float> cs_want = to_dev_layout(cs_want_ref);
// THE LAYOUT: the manifest's `ssm_conv1d.weight` has ne = [d_conv, C], so ne0 = d_conv is fast.
// The kernel takes `kW[c*dc + i]`; the reference's `kern` is (dc, C) row-major. The transcription
// below is the ONLY place the two meet.
std::vector<float> kW((size_t) C * dc);
for (int c = 0; c < C; ++c)
for (int i = 0; i < dc; ++i) kW[(size_t) (c * dc + i)] = kern[(size_t) (i * C + c)];
// and the rival reading, which must differ or the fixture cannot see a transposed weight
std::vector<float> kW_wrong((size_t) C * dc);
for (size_t i = 0; i < kern.size(); ++i) kW_wrong[i] = kern[i];
{
// apply the WRONG layout by hand: tap i of channel c is kW[c*dc+i] read as kW[i*C+c]
std::vector<float> cs2 = cs;
std::vector<float> out_bad((size_t) C, 0.0f);
for (int c = 0; c < C; ++c) {
double a = 0;
for (int i = 0; i < dc - 1; ++i) a += (double) cs2[(size_t) (i * C + c)] * (double) kW_wrong[(size_t) (c * dc + i)];
a += (double) x[(size_t) c] * (double) kW_wrong[(size_t) (c * dc + dc - 1)];
out_bad[(size_t) c] = (float) a;
}
const double rel = rel_l1(want, out_bad);
std::printf("\n %-42s %-4s (%.1f%% apart)\n", "conv weight layout is observable",
rel > 0.05 ? "yes" : "*** NO ***", rel * 100);
if (!(rel > 0.05)) ++bad;
}
float *d_cs = nullptr, *d_x = nullptr, *d_w = nullptr, *d_o = nullptr;
check(cudaMalloc(&d_cs, cs.size() * 4), "cs");
check(cudaMalloc(&d_x, x.size() * 4), "cx");
check(cudaMalloc(&d_w, kW.size() * 4), "cw");
check(cudaMalloc(&d_o, (size_t) C * 4), "co");
check(cudaMemcpy(d_cs, cs.data(), cs.size() * 4, cudaMemcpyHostToDevice), "ccs");
check(cudaMemcpy(d_x, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "ccx");
check(cudaMemcpy(d_w, kW.data(), kW.size() * 4, cudaMemcpyHostToDevice), "ccw");
strata::kernels::gdn_conv_step(d_cs, d_x, d_w, d_o, C, dc, nullptr);
std::vector<float> got((size_t) C), got_cs(cs.size());
check(cudaMemcpy(got.data(), d_o, got.size() * 4, cudaMemcpyDeviceToHost), "cgo");
check(cudaMemcpy(got_cs.data(), d_cs, got_cs.size() * 4, cudaMemcpyDeviceToHost), "cgs");
const double rel_out = rel_l1(want, got), rel_cs = rel_l1(cs_want, got_cs);
std::printf(" %-42s rel %.3e\n", "conv out vs reference", rel_out);
std::printf(" %-42s rel %.3e\n", "conv state slide (mapped) vs reference", rel_cs);
if (!(rel_out <= 1e-6)) { std::printf(" *** conv out ***\n"); ++bad; }
if (!(rel_cs <= 1e-6)) { std::printf(" *** conv state ***\n"); ++bad; }
cudaFree(d_cs); cudaFree(d_x); cudaFree(d_w); cudaFree(d_o);
}
// ================= 3. l2_norm =================
{
const int rows = 16, cols = 128;
std::vector<float> x((size_t) rows * cols), x_ref;
for (auto& val : x) val = gauss(rng) * 3.0f;
x_ref = x;
const float eps = 1e-6f;
for (int r = 0; r < rows; ++r) {
double acc = 0;
for (int i = 0; i < cols; ++i) acc += (double) x_ref[(size_t) (r * cols + i)] * (double) x_ref[(size_t) (r * cols + i)];
const float inv = (float) (1.0 / std::sqrt(acc + (double) eps));
for (int i = 0; i < cols; ++i) x_ref[(size_t) (r * cols + i)] *= inv;
}
// The rival reading puts THE SAME eps on the MEAN instead of on the squared norm, which at
// ||x||^2 ~ 1152 and cols = 128 changes the divisor from sqrt(1152) to sqrt(9) - a factor of 11.3, so
// the trap is large here. It would be a much smaller correction on a near-zero row, where the floor
// is the whole answer; that regime is not what this fixture exercises.
std::vector<float> x_mean = x;
for (int r = 0; r < rows; ++r) {
double acc = 0;
for (int i = 0; i < cols; ++i) acc += (double) x_mean[(size_t) (r * cols + i)] * (double) x_mean[(size_t) (r * cols + i)];
const float inv = (float) (1.0 / std::sqrt(acc / cols + (double) eps));
for (int i = 0; i < cols; ++i) x_mean[(size_t) (r * cols + i)] *= inv;
}
const double rel_trap = rel_l1(x_ref, x_mean);
std::printf("\n %-42s %-4s (%.1f%% apart)\n", "eps on sum vs on mean is observable",
rel_trap > 0.05 ? "yes" : "*** NO ***", rel_trap * 100);
if (!(rel_trap > 0.05)) ++bad;
float* d_x = nullptr;
check(cudaMalloc(&d_x, x.size() * 4), "lx");
check(cudaMemcpy(d_x, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "lcx");
strata::kernels::gdn_l2_norm(d_x, rows, cols, eps, nullptr);
std::vector<float> got(x.size());
check(cudaMemcpy(got.data(), d_x, got.size() * 4, cudaMemcpyDeviceToHost), "lcg");
const double rel = rel_l1(x_ref, got);
std::printf(" %-42s rel %.3e\n", "l2_norm vs reference", rel);
if (!(rel <= 1e-6)) ++bad;
cudaFree(d_x);
}
// ================= 4. the closing norm =================
{
const int S2 = 128, hv2 = 8;
std::vector<float> o((size_t) hv2 * S2), z((size_t) hv2 * S2), sn((size_t) S2), y_ref((size_t) hv2 * S2);
for (auto& val : o) val = gauss(rng);
for (auto& val : z) val = gauss(rng);
for (auto& val : sn) val = 1.0f + 0.1f * gauss(rng);
const float eps = 1e-6f;
for (int h = 0; h < hv2; ++h) {
double acc = 0;
for (int i = 0; i < S2; ++i) acc += (double) o[(size_t) (h * S2 + i)] * (double) o[(size_t) (h * S2 + i)];
const float inv = (float) (1.0 / std::sqrt(acc / S2 + (double) eps));
for (int i = 0; i < S2; ++i)
y_ref[(size_t) (h * S2 + i)] = o[(size_t) (h * S2 + i)] * inv * sn[(size_t) i] *
sigmoid_f(z[(size_t) (h * S2 + i)]);
}
float *d_o = nullptr, *d_z = nullptr, *d_sn = nullptr, *d_y = nullptr;
check(cudaMalloc(&d_o, o.size() * 4), "no");
check(cudaMalloc(&d_z, z.size() * 4), "nz");
check(cudaMalloc(&d_sn, sn.size() * 4), "ns");
check(cudaMalloc(&d_y, y_ref.size() * 4), "ny");
check(cudaMemcpy(d_o, o.data(), o.size() * 4, cudaMemcpyHostToDevice), "nco");
check(cudaMemcpy(d_z, z.data(), z.size() * 4, cudaMemcpyHostToDevice), "ncz");
check(cudaMemcpy(d_sn, sn.data(), sn.size() * 4, cudaMemcpyHostToDevice), "ncs");
strata::kernels::gdn_out_norm(d_o, d_z, d_sn, d_y, hv2, S2, eps, nullptr);
std::vector<float> got(y_ref.size());
check(cudaMemcpy(got.data(), d_y, got.size() * 4, cudaMemcpyDeviceToHost), "ncy");
const double rel = rel_l1(y_ref, got);
std::printf("\n %-42s rel %.3e\n", "out norm (rms * ssm_norm * sigmoid z)", rel);
if (!(rel <= 1e-6)) ++bad;
// SiLU instead of sigmoid is the qwen3.5 reading; `ref/gdn.py` says this artifact does NOT use it
std::vector<float> y_silu(y_ref.size());
for (int h = 0; h < hv2; ++h) {
double acc = 0;
for (int i = 0; i < S2; ++i) acc += (double) o[(size_t) (h * S2 + i)] * (double) o[(size_t) (h * S2 + i)];
const float inv = (float) (1.0 / std::sqrt(acc / S2 + (double) eps));
for (int i = 0; i < S2; ++i) {
const float zz = z[(size_t) (h * S2 + i)];
y_silu[(size_t) (h * S2 + i)] =
o[(size_t) (h * S2 + i)] * inv * sn[(size_t) i] * (zz / (1.0f + std::exp(-zz)));
}
}
const double rel_silu = rel_l1(y_ref, y_silu);
std::printf(" %-42s %-4s (%.2f%% apart)\n", "sigmoid vs SiLU on the gate is observable",
rel_silu > 0.05 ? "yes" : "*** NO ***", rel_silu * 100);
if (!(rel_silu > 0.05)) ++bad;
cudaFree(d_o); cudaFree(d_z); cudaFree(d_sn); cudaFree(d_y);
}
std::printf("\ngdn: %d failures\n", bad);
if (bad) return 1;
if (selftest) std::printf("gdn_parity OK\n");
return 0;
}
|