Logic pass: CPU attention mirrors get the same int32 bound as the GPU kernels (peers refuse identically); gpuMlpChain returns pooled buffers when the row-max guard throws
Browse files- web/TEST_RESULTS.md +1 -1
- web/public/verified_core.js +4 -0
- web/public/webgpu.js +6 -1
- web/test_reductions.js +6 -0
web/TEST_RESULTS.md
CHANGED
|
@@ -18,7 +18,7 @@ against the verified units at init.
|
|
| 18 |
| `test_unit_backward.js` | int8 STE gradients do not damage convergence | PASS — units/float loss ratio 1.007 |
|
| 19 |
| `test_b2b.js` | the chained GPU MLP's CPU mirror: gemm1 byte-identical to the un-chained GEMM, deterministic, +0.5 quantize FMA-contraction-immune | PASS — chained transformer converges 4.7384 → 2.0512 |
|
| 20 |
| `test_nonfinite.js` | every quantizer refuses NaN/Inf instead of zeroing the layer | PASS — 13/13; the old quantizers fail 12 (added 2026-09-27) |
|
| 21 |
-
| `test_reductions.js` | the GPU row-max guard is complete, `averageGrads` refuses wrong-length / non-finite gradients, int32 accumulators refuse K > 131,071 | PASS —
|
| 22 |
|
| 23 |
## The IEEE-754 oracle (`test_ieee.js`)
|
| 24 |
|
|
|
|
| 18 |
| `test_unit_backward.js` | int8 STE gradients do not damage convergence | PASS — units/float loss ratio 1.007 |
|
| 19 |
| `test_b2b.js` | the chained GPU MLP's CPU mirror: gemm1 byte-identical to the un-chained GEMM, deterministic, +0.5 quantize FMA-contraction-immune | PASS — chained transformer converges 4.7384 → 2.0512 |
|
| 20 |
| `test_nonfinite.js` | every quantizer refuses NaN/Inf instead of zeroing the layer | PASS — 13/13; the old quantizers fail 12 (added 2026-09-27) |
|
| 21 |
+
| `test_reductions.js` | the GPU row-max guard is complete, `averageGrads` refuses wrong-length / non-finite gradients, int32 accumulators refuse K > 131,071 | PASS — 19/19; the old code fails 13 (added 2026-09-30) |
|
| 22 |
|
| 23 |
## The IEEE-754 oracle (`test_ieee.js`)
|
| 24 |
|
web/public/verified_core.js
CHANGED
|
@@ -447,6 +447,9 @@
|
|
| 447 |
// d.acc=true returns the raw int32 accumulator (exact oracle for the kernel gate)
|
| 448 |
function attScoresJS(qq, kq, qs, ks, d, L) {
|
| 449 |
const { B, T, heads, hd } = d, C = heads * hd, mul = L.mul, raw = !!d.acc;
|
|
|
|
|
|
|
|
|
|
| 450 |
const out = raw ? new Int32Array(B * heads * T * T) : new Float32Array(B * heads * T * T);
|
| 451 |
for (let bi = 0; bi < B; bi++) for (let h = 0; h < heads; h++) {
|
| 452 |
const bz = bi * heads + h;
|
|
@@ -465,6 +468,7 @@
|
|
| 465 |
// ctx[(bi,ti),(h,j)] = Σ_tj a[bz,ti,tj]·v[(bi,tj),(h,j)] — scatter fused into BT×C
|
| 466 |
function attCtxJS(aq, vq, as, vs, d, L) {
|
| 467 |
const { B, T, heads, hd } = d, C = heads * hd, mul = L.mul, raw = !!d.acc;
|
|
|
|
| 468 |
const out = raw ? new Int32Array(B * T * C) : new Float32Array(B * T * C);
|
| 469 |
for (let bi = 0; bi < B; bi++) for (let h = 0; h < heads; h++) {
|
| 470 |
const bz = bi * heads + h;
|
|
|
|
| 447 |
// d.acc=true returns the raw int32 accumulator (exact oracle for the kernel gate)
|
| 448 |
function attScoresJS(qq, kq, qs, ks, d, L) {
|
| 449 |
const { B, T, heads, hd } = d, C = heads * hd, mul = L.mul, raw = !!d.acc;
|
| 450 |
+
// the WGSL kernel sums these in i32; this mirror sums in doubles. Bound both identically so
|
| 451 |
+
// a CPU peer and a GPU peer refuse the same shapes instead of one halting alone.
|
| 452 |
+
accBound(hd, "attScoresJS");
|
| 453 |
const out = raw ? new Int32Array(B * heads * T * T) : new Float32Array(B * heads * T * T);
|
| 454 |
for (let bi = 0; bi < B; bi++) for (let h = 0; h < heads; h++) {
|
| 455 |
const bz = bi * heads + h;
|
|
|
|
| 468 |
// ctx[(bi,ti),(h,j)] = Σ_tj a[bz,ti,tj]·v[(bi,tj),(h,j)] — scatter fused into BT×C
|
| 469 |
function attCtxJS(aq, vq, as, vs, d, L) {
|
| 470 |
const { B, T, heads, hd } = d, C = heads * hd, mul = L.mul, raw = !!d.acc;
|
| 471 |
+
accBound(T, "attCtxJS"); // contracts over keys, like gpuAttCtx
|
| 472 |
const out = raw ? new Int32Array(B * T * C) : new Float32Array(B * T * C);
|
| 473 |
for (let bi = 0; bi < B; bi++) for (let h = 0; h < heads; h++) {
|
| 474 |
const bz = bi * heads + h;
|
web/public/webgpu.js
CHANGED
|
@@ -856,7 +856,12 @@
|
|
| 856 |
const mx = new Float32Array(readM.getMappedRange(0, m * 4).slice(0)); readM.unmap();
|
| 857 |
// scale derivation in JS f64 — exactly rounded, identical on every device
|
| 858 |
// (WGSL division is 2.5 ULP, which is why it never runs on the GPU)
|
| 859 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 860 |
const bufInv = up(device, sc.inv, SU), bufHs = up(device, sc.scale, SU);
|
| 861 |
const bufQ = mk(device, m * hw * 4, GPUBufferUsage.STORAGE);
|
| 862 |
const bufDQ = up(device, new Uint32Array([m, h, hw, 0]), UU);
|
|
|
|
| 856 |
const mx = new Float32Array(readM.getMappedRange(0, m * 4).slice(0)); readM.unmap();
|
| 857 |
// scale derivation in JS f64 — exactly rounded, identical on every device
|
| 858 |
// (WGSL division is 2.5 ULP, which is why it never runs on the GPU)
|
| 859 |
+
let sc;
|
| 860 |
+
try { sc = root.Verified.scalesFromAbsMax(mx); } // refuses a non-finite row max
|
| 861 |
+
catch (e) { // hand the pooled buffers back first
|
| 862 |
+
release(device, [bufX, bufW1, bufW2, bufRs, bufCs1, bufCs2, bufH, bufMX, bufD1, bufDM, readH, readM]);
|
| 863 |
+
throw e;
|
| 864 |
+
}
|
| 865 |
const bufInv = up(device, sc.inv, SU), bufHs = up(device, sc.scale, SU);
|
| 866 |
const bufQ = mk(device, m * hw * 4, GPUBufferUsage.STORAGE);
|
| 867 |
const bufDQ = up(device, new Uint32Array([m, h, hw, 0]), UU);
|
web/test_reductions.js
CHANGED
|
@@ -77,6 +77,12 @@ ck("(context) Int32Array really wraps 2^31 to -2^31 -- the silent failure being
|
|
| 77 |
ck("bgemmJS refuses the same K",
|
| 78 |
throwsName(() => V.bgemmJS(worst, worst, new Float32Array(1), new Float32Array(1),
|
| 79 |
{ m: 1, k: K, n: 1 }, L), "AccumulatorBoundError"));
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
const okK = worst.subarray(0, V.MAX_INT32_K);
|
| 81 |
ck("K = 131071 still runs and is exact (131071 * 16384)",
|
| 82 |
V.lutMatmulJS(okK, okK, 1, V.MAX_INT32_K, 1, L)[0] === 131071 * 16384);
|
|
|
|
| 77 |
ck("bgemmJS refuses the same K",
|
| 78 |
throwsName(() => V.bgemmJS(worst, worst, new Float32Array(1), new Float32Array(1),
|
| 79 |
{ m: 1, k: K, n: 1 }, L), "AccumulatorBoundError"));
|
| 80 |
+
ck("attScoresJS refuses head width hd > 131071 (the CPU mirror of gpuAttScores)",
|
| 81 |
+
throwsName(() => V.attScoresJS(worst, worst, new Float32Array(1), new Float32Array(1),
|
| 82 |
+
{ B: 1, T: 1, heads: 1, hd: K }, L), "AccumulatorBoundError"));
|
| 83 |
+
ck("attCtxJS refuses T > 131071 (it contracts over keys, like gpuAttCtx)",
|
| 84 |
+
throwsName(() => V.attCtxJS(new Int8Array(1), new Int8Array(1), new Float32Array(1), new Float32Array(1),
|
| 85 |
+
{ B: 1, T: K, heads: 1, hd: 1 }, L), "AccumulatorBoundError"));
|
| 86 |
const okK = worst.subarray(0, V.MAX_INT32_K);
|
| 87 |
ck("K = 131071 still runs and is exact (131071 * 16384)",
|
| 88 |
V.lutMatmulJS(okK, okK, 1, V.MAX_INT32_K, 1, L)[0] === 131071 * 16384);
|