Probe failure halts, fragment cleanup, subnormal fgemm gate, live f32 audit, high-performance adapter (2026-10-14)

public/app.js

@@ -268,15 +268,22 @@ function setupChannel(peerId, dc) {
     // the next step boundary instead of making them wait for the next run
     if (training && leaderId === null) { pendingJoins.add(peerId); log(`${nmeOf(peerId)} will be synced into the running training`); }
   };
-  dc.onclose = () => { chans.delete(peerId); updatePeers(); wake(); scheduleReconnect(peerId); };
+  dc.onclose = () => { chans.delete(peerId); dropFragments(peerId); updatePeers(); wake(); scheduleReconnect(peerId); };
   dc.onmessage = (e) => onGrad(peerId, e.data);
 }
 function cleanupPeer(id) {
   const pc = pcs.get(id); if (pc) pc.close();
   pcs.delete(id); chans.delete(id); pendingCand.delete(id); pendingJoins.delete(id);
-  for (const k of fragIn.keys()) if (k.startsWith(id + ":")) fragIn.delete(k);
+  dropFragments(id);
   wake();
 }
+// A message cut off by a closed channel is never completed (the sender does not
+// resume it), so its parts must go when the channel does. Before 2026-10-14 only
+// cleanupPeer dropped them, and only the reconnect INITIATOR calls that: the
+// other side kept up to a resume bundle's worth of dead parts per drop.
+function dropFragments(id) {
+  for (const k of [...fragIn.keys()]) if (k.startsWith(id + ":")) fragIn.delete(k);
+}
 
 // ---- checkpoints ------------------------------------------------------------
 // File layout (also the broadcast payload after the sentinel):
@@ -555,8 +562,13 @@ function onFragment(peerId, buf) {               // returns full message when co
   const key = peerId + ":" + id;
   let st = fragIn.get(key);
   if (!st) { st = { id, parts: [], got: 0, total }; fragIn.set(key, st); }
+  if (seq < 0 || seq >= st.total || total !== st.total) {     // malformed: drop the whole message, loudly
+    log(`dropped malformed message ${id} from ${nmeOf(peerId)} (fragment ${seq}/${total}, expected /${st.total})`);
+    fragIn.delete(key);
+    return null;
+  }
+  if (!st.parts[seq]) st.got++;                  // a repeated part must not count twice
   st.parts[seq] = new Uint8Array(buf, 16).slice(0);
-  st.got++;
   if (st.got < st.total) return null;
   fragIn.delete(key);
   let len = 0; for (const p of st.parts) len += p.length;
@@ -612,8 +624,12 @@ async function refreshProbe() {
     mathHash = Compute.mathProbe();
     log(`JS math library hash: ${mathHash.toString(16)} (peers must match; ECMA-262 does not require exp/log/cos/sin to be correctly rounded)`);
   }
+  // A failed probe must not leave the PREVIOUS hash in place: that hash would
+  // go out with every gradient as a proof of kernels this run never re-checked.
+  // 0 means "unproven", and the roster check halts on it (2026-10-14; the same
+  // shape as an emulator call that reported success without doing the work).
   try { probeHash = await Compute.kernelProbe(compute, L); }
-  catch (e) { log(`kernel probe failed to run: ${e.message}`); }
+  catch (e) { probeHash = 0; log(`kernel probe failed to run: ${e.message} — this device's kernels are UNPROVEN`); }
   return probeHash;
 }
 function hashWeights() {                        // FNV-1a over the flat param bytes
@@ -898,8 +914,16 @@ async function train(cfg, resume) {
       if (bad) { halted = `weights diverged from ${nmeOf(bad)} (detected at step ${s + 1})`; break; }
       // kernel disagreement: same seeded probe, same exact integer arithmetic
       // expected on every backend. This is the check the weight hash can't make.
-      const badK = roster.find(id => id !== myId && hs.has(id) && hs.get(id).phash &&
-                                     probeHash && hs.get(id).phash !== probeHash);
+      // phash 0 = that device's probe did not run. It used to make this
+      // comparison skip the peer (`phash && ...`), so a device whose kernels
+      // could not be proven trained on unchecked; now it stops the run.
+      const unproven = roster.find(id => hs.has(id) && !hs.get(id).phash);
+      if (unproven) {
+        halted = `${unproven === myId ? "this device's" : nmeOf(unproven) + "'s"} kernel probe failed to run ` +
+                 `at step ${s + 1} — its arithmetic is unproven, so its gradients cannot be trusted`;
+        break;
+      }
+      const badK = roster.find(id => id !== myId && hs.has(id) && hs.get(id).phash !== probeHash);
       if (badK) {
         halted = `kernel disagreement with ${nmeOf(badK)} at step ${s + 1} — their verified-unit probe ` +
                  `hashes ${hs.get(badK).phash} vs mine ${probeHash}. One of us is computing different ` +

public/transformer.js

@@ -182,7 +182,7 @@
       cfg: { ...cfg, layers, heads, hidden, vocab: vocabSize() },
       ctx: { L, bgemm: (engine && engine.bgemm) || null,
              att: (engine && engine.att) || null, fgemm: (engine && engine.fgemm) || null,
-             fgemm2: (engine && engine.fgemm2) || null,
+             fgemm2: (engine && engine.fgemm2) || null, fgemmFma: !!(engine && engine.fgemmFma),
              mlp: (engine && engine.mlp) || null,
              audit: audit || null, unitBackward: !!cfg.unitBackward },
       emb: add("emb", mk(vocabSize() * c, 0.08)),
@@ -425,6 +425,14 @@
         bmm(cache.dlogits, m.emb, BT, vocab, C),
       ]);
     }
+    // live-shape audit of the f32 GEMM pair (both legs; gated only at small shapes)
+    if ((m.ctx.fgemm2 || m.ctx.fgemm) && !units && m.ctx.audit && m.ctx.audit.due()) {
+      const bad = V.auditFgemm(cache.dlogits, cache.lnf.y, { m: vocab, k: BT, n: C, transA: true },
+                               g[gi.emb], m.ctx.fgemmFma, m.ctx.audit.cells) ||
+                  V.auditFgemm(cache.dlogits, m.emb, { m: BT, k: vocab, n: C },
+                               dlnfIn, m.ctx.fgemmFma, m.ctx.audit.cells);
+      if (bad) m.ctx.audit.fail(bad);
+    }
     let dx = lnBwd(dlnfIn, cache.lnf.y, cache.lnf.sig, BT, C);
     const scale = 1 / Math.sqrt(hd);
     // concat helper for fusing sibling GEMMs into one batched dispatch

public/verified_core.js

@@ -334,6 +334,40 @@
     return out;
   }
 
+  // ---- live audit for the split-K f32 GEMM -----------------------------------
+  // Every int8 kernel had a live-shape audit; the f32 backward GEMM -- whose
+  // output IS the embedding gradient -- had only its init gate at 7x4500x5 and
+  // 9x300x6, while training runs it at vocab x BT x C. Same remedy as auditTile:
+  // recompute a few cells (structural corners first) with the mirror's exact
+  // partition and rounding schedule, compare bit-for-bit (2026-10-14 review).
+  function fgemmCell(A, Bm, d, fma, row, col) {
+    const { m, k, n } = d, transA = !!d.transA;
+    const S = k > 4096 ? Math.min(16, Math.ceil(k / 2048)) : 1;
+    const ks = Math.ceil(k / S);
+    let acc = 0;
+    for (let z = 0; z < S; z++) {
+      const p0 = z * ks, p1 = Math.min(k, p0 + ks);
+      let s = 0;
+      for (let p = p0; p < p1; p++) {
+        const a = transA ? A[p * m + row] : A[row * k + p];
+        s = fma ? f32(s + a * Bm[p * n + col]) : f32(s + f32(a * Bm[p * n + col]));
+      }
+      acc = f32(acc + s);
+    }
+    return acc;
+  }
+  function auditFgemm(A, Bm, d, got, fma, nCells) {
+    const { m, n } = d, N = nCells || 8;
+    const edges = [[m - 1, n - 1], [0, n - 1], [m - 1, 0], [0, 0]];
+    for (let t = 0; t < N; t++) {
+      const [i, j] = t < edges.length ? edges[t] : [(Math.random() * m) | 0, (Math.random() * n) | 0];
+      const v = fgemmCell(A, Bm, d, fma, i, j);
+      if (bitDiff(got[i * n + j], v))
+        return `f32 GEMM audit failed at [${i},${j}] shape ${m}x${d.k}x${n}${d.transA ? " transA" : ""}: kernel ${got[i * n + j]} vs mirror ${v}`;
+    }
+    return null;
+  }
+
   // ---- live audits for the fused attention kernels ---------------------------
   // The attention kernels had exact INIT gates but nothing at live shapes —
   // the exact gap the GEMM audit exists to close, left open on the kernels with
@@ -538,7 +572,7 @@
   const api = { assertFinite, accBound, MAX_INT32_K, quantize, quantize2, quantizeRows, quantizeCols, quantizeHeadCols, lutMatmulJS, lutMatmul3JS, lutMatmul3,
                 bgemmJS, vgemmBlock, auditTile, epi, attScoresJS, attCtxJS, linearFwd, forward, backward, splitApply,
                 rowAbsMax, scalesFromAbsMax, quantizeRowsInv, vmlpBlock, bitDiff,
-                auditAttScores, auditAttCtx, fgemmMirror };
+                auditAttScores, auditAttCtx, fgemmMirror, auditFgemm };
   if (typeof module !== "undefined" && module.exports) { TC = require("./traincore.js"); module.exports = api; }
   else { TC = root.TrainCore; root.Verified = api; }
 })(typeof self !== "undefined" ? self : this);

public/webgpu.js

@@ -286,7 +286,11 @@
                   matmulInt8: (Xq, Wq, m, k, n, LL) => root.Verified.lutMatmulJS(Xq, Wq, m, k, n, LL) };
     if (!(root.navigator && navigator.gpu)) return cpu;
     try {
-      const adapter = await navigator.gpu.requestAdapter();
+      // Ask for the discrete GPU: the default adapter on a laptop is the iGPU
+      // (measured 2026-10-14: Intel gen-9 picked while an RTX sat idle). The
+      // browser may still hand back the iGPU; the gates decide what runs.
+      const adapter = (await navigator.gpu.requestAdapter({ powerPreference: "high-performance" })) ||
+                      (await navigator.gpu.requestAdapter());
       if (!adapter) return cpu;
       const device = await adapter.requestDevice();
       const info = adapter.info || {};
@@ -330,9 +334,27 @@
       // because head-strided addressing is where these kernels can go wrong.
       let att = { scores: (qq, kq, qs, ks, d) => gpuAttScores(device, d.acc ? scoresVPipe : scoresPipe, qq, kq, qs, ks, d),
                   ctx: (aq, vq, as, vs, d) => gpuAttCtx(device, d.acc ? ctxVPipe : ctxPipe, aq, vq, as, vs, d) };
+      // Two sweeps: the second runs on RECYCLED pool buffers left full of
+      // large-magnitude residue (same shapes => same buckets, LIFO free list),
+      // the dirty-buffer gate the bgemm and MLP kernels already had. Without it
+      // a kernel that skips writing some output cell would pass on fresh
+      // buffers and corrupt live steps (2026-10-14 review).
+      const ATT_SHAPES = [{ B: 2, T: 8, heads: 2, hd: 8 }, { B: 1, T: 32, heads: 2, hd: 16 },
+                          { B: 3, T: 7, heads: 3, hd: 5 }, { B: 2, T: 33, heads: 4, hd: 8 }];
       try {
-        for (const d0 of [{ B: 2, T: 8, heads: 2, hd: 8 }, { B: 1, T: 32, heads: 2, hd: 16 },
-                          { B: 3, T: 7, heads: 3, hd: 5 }, { B: 2, T: 33, heads: 4, hd: 8 }]) {
+        for (const dirty of [false, true]) {
+          if (dirty) for (const d0 of ATT_SHAPES) {
+            const nQ = d0.B * d0.T * d0.heads * d0.hd, nA = d0.B * d0.heads * d0.T * d0.T;
+            const big = (n) => new Float32Array(n).fill(1e4);
+            for (const acc of [false, true]) {
+              const dv = { ...d0, acc };
+              await att.scores(new Int8Array(nQ).fill(127), new Int8Array(nQ).fill(127),
+                               big(d0.B * d0.T * d0.heads), big(d0.B * d0.T * d0.heads), dv);
+              await att.ctx(new Int8Array(nA).fill(127), new Int8Array(nQ).fill(127),
+                            big(d0.B * d0.heads * d0.T), big(d0.B * d0.heads * d0.hd), dv);
+            }
+          }
+        for (const d0 of ATT_SHAPES) {
           const nQ = d0.B * d0.T * d0.heads * d0.hd;
           const qq = new Int8Array(nQ), kq = new Int8Array(nQ), vq = new Int8Array(nQ);
           for (let i = 0; i < nQ; i++) { qq[i] = (Math.random() * 256 - 128) | 0; kq[i] = (Math.random() * 256 - 128) | 0; vq[i] = (Math.random() * 256 - 128) | 0; }
@@ -353,7 +375,8 @@
           const refS = root.Verified.attScoresJS(qq, kq, qs, ks, d0, L);
           const refC = root.Verified.attCtxJS(aq, vq, as, vs, d0, L);
           for (let i = 0; i < refS.length; i++) if (bitDiff(hwS[i], refS[i])) throw new Error(`scores epilogue mismatch @${i}`);
-          for (let i = 0; i < refC.length; i++) if (bitDiff(hwC[i], refC[i])) throw new Error(`ctx epilogue mismatch @${i}`);
+          for (let i = 0; i < refC.length; i++) if (bitDiff(hwC[i], refC[i])) throw new Error(`ctx epilogue mismatch @${i}${dirty ? " (dirty pool)" : ""}`);
+        }
         }
       } catch (e) {
         console.warn("fused attention kernels failed verification — using CPU LUT mirrors:", e.message);
@@ -398,6 +421,39 @@
         const Bt = Float32Array.from({ length: mT * nT }, () => Math.random() - 0.5);
         const Bn = Float32Array.from({ length: kT * nT }, () => Math.random() - 0.5);
         const dT = { m: kT, k: mT, n: nT, transA: true }, dN = { m: mT, k: kT, n: nT };
+        // transA against the MIRROR, not just against fgemm: before 2026-10-14
+        // the transA index pattern was only ever compared with itself (fgemm2
+        // leg vs fgemm call, same shader), so a wrong A[p*m+row] passed. The
+        // second shape puts transA through split-K (k > 4096) as well.
+        const fma = fgemmFma;
+        for (const [mA, kA, nA] of [[kT, mT, nT], [6, 4500, 3]]) {
+          const At = Float32Array.from({ length: mA * kA }, () => Math.random() - 0.5);
+          const Bx = Float32Array.from({ length: kA * nA }, () => Math.random() - 0.5);
+          const dX = { m: mA, k: kA, n: nA, transA: true };
+          const got = await fgemm(At, Bx, dX), want = root.Verified.fgemmMirror(At, Bx, dX, fma);
+          for (let i = 0; i < want.length; i++) if (bitDiff(got[i], want[i])) throw new Error(`fgemm transA differs from the mirror @${i} (${mA}x${kA}x${nA})`);
+        }
+        // SUBNORMALS: WGSL permits an implementation to flush f32 subnormals to
+        // zero, and the JS mirror never flushes. The random inputs above are
+        // ~0.1 in magnitude and never produce one, so a flushing GPU passed this
+        // gate and then forked from non-flushing peers on tiny gradients
+        // (2026-10-14; the emulator's FTZ/DAZ lesson). Products here are ~1e-40.
+        {
+          const dS = { m: 3, k: 64, n: 2 };
+          const As = Float32Array.from({ length: dS.m * dS.k }, (_, i) => (i % 2 ? 1 : -1) * (1 + (i % 7)) * 1e-20);
+          const Bs = Float32Array.from({ length: dS.k * dS.n }, (_, i) => (1 + (i % 5)) * 1e-20);
+          const got = await fgemm(As, Bs, dS), want = root.Verified.fgemmMirror(As, Bs, dS, fma);
+          let sub = 0;
+          for (let i = 0; i < want.length; i++) if (want[i] !== 0 && Math.abs(want[i]) < 1.1754943508222875e-38) sub++;
+          if (!sub) throw new Error("subnormal gate case produced no subnormal (test is vacuous)");
+          for (let i = 0; i < want.length; i++) if (bitDiff(got[i], want[i]))
+            throw new Error(`fgemm flushes or rounds subnormals differently @${i}: ${got[i]} vs mirror ${want[i]}`);
+        }
+        // dirty re-gate: poison the partial/output buckets at the gate's shapes, then re-check
+        await fgemm(new Float32Array(m0 * k0).fill(1e4), new Float32Array(k0 * n0).fill(1e4), dG);
+        const hw2 = await fgemm(A, Bm, dG);
+        const mRef = fma ? mFma : mStep;
+        for (let i = 0; i < hw2.length; i++) if (bitDiff(hw2[i], mRef[i])) throw new Error(`fgemm differs on a recycled (dirty) pool @${i}`);
         const [f1, f2] = await fgemm2(A2, Bt, dT, Bn, dN);
         const [r1, r2] = [await fgemm(A2, Bt, dT), await fgemm(A2, Bn, dN)];
         for (let i = 0; i < r1.length; i++) if (bitDiff(f1[i], r1[i])) throw new Error("fgemm2 transA leg differs");
@@ -559,7 +615,7 @@
 
       const viaLUT = { backend: "webgpu", label: `${gpuName} (LUT shader · exact-gated)`,
                        matmulInt8: (Xq, Wq, m, k, n) => gpuMatmulLUT(device, lutPipe, lutBuf, Xq, Wq, m, k, n),
-                       bgemm: bgLut, att, fgemm, fgemm2, mlp: mlpLut };
+                       bgemm: bgLut, att, fgemm, fgemm2, fgemmFma, mlp: mlpLut };
 
       // DP4A pipeline — only if the WGSL feature exists AND its batched kernel
       // reproduces the verified units exactly across the shape sweep
@@ -587,7 +643,7 @@
       // worse than a fixed choice. DP4A ships: it wins the int8-backward mode
       // by ~12% and ties elsewhere.
       return { backend: "webgpu", label: `${gpuName} (DP4A int8 dot · exact-gated vs units)`,
-               bgemm: bg, att, fgemm, fgemm2, mlp: mlpDp4 };
+               bgemm: bg, att, fgemm, fgemm2, fgemmFma, mlp: mlpDp4 };
     } catch (e) { console.warn("WebGPU init failed, CPU fallback:", e); return cpu; }
   }