weights.js

/* weights.js — GGUF *weights* stored in HDC memory, and used for a forward pass.
 *
 * The engine in `vsa.js` reads a GGUF's vocabulary out of the metadata block and builds an
 * association memory from the merge table and from text. Its weights were previously discarded,
 * because the legacy page folded raw weight bytes into a D=10,000 hypervector by a monotone OR and
 * measured at chance (15.6% against a 12.5% floor) — 1,185,376 bytes collapsed into 10,000
 * dimensions, a 948:1 lossy reduction whose information did not survive.
 *
 * This file does the three things that actually make weight storage in HDC memory work, each
 * grounded in published work and separately measurable:
 *
 *  1. DEQUANTISE real GGUF tensors exactly (F32/F16/Q4_0/Q4_1/Q5_0/Q5_1/Q8_0/Q6_K/IQ4_NL, plus the
 *     K-quants Q2_K/Q3_K/Q4_K/Q5_K and BF16). The block arithmetic below reproduces the archive's
 *     verified reference implementations case for case — `D:/JS-GGUF/GGUF-SURGERY/gguf-core.js`
 *     `dequantTensor` for every quantised type, and its `bf16ToF32` for BF16 (BF16 is the top 16
 *     bits of an IEEE-754 single, so it needs no block loop). Nothing is guessed: a type with no
 *     reference to port stays unsupported and returns null rather than a plausible-looking wrong
 *     number. That currently includes Q8_1 (9) and MXFP4 (39), for which the archive has no
 *     implementation.
 *
 *  2. LOSSLESS paged HDC weight memory. `D:/JS-GGUF/HDC-PROOFS/wan_hdc_weight_memory.py` states the
 *     archive's own hard-won conclusion: "Arbitrary FP16/BF16 model bytes cannot be losslessly
 *     represented by a fixed-size semantic hypervector. Instead, each source byte is represented by
 *     eight bipolar dimensions ... The page dimension is therefore N = byte_count * 8 and scales
 *     with every artifact." That is implemented here: one dimension per stored BIT, paged, and
 *     byte-exact on restore. It is a *representation*, not a compression — its dimension grows with
 *     the data, and saying otherwise is the error this file exists to avoid.
 *
 *  3. TERNARY storage, which IS compression, per arXiv 2402.17764 (Ma et al., "The Era of 1-bit
 *     LLMs: BitNet b1.58") — every weight becomes one of {-1,0,+1} as W = s * T with s = absmean.
 *     A ternary row is stored as two bipolar bit-planes (which weights are +1, which are -1), so the
 *     cost is 2 bits per weight plus one scale per block, and the reconstruction error against the
 *     original is MEASURED by `ternaryError()` rather than claimed.
 *
 *  4. A VSA LINEAR LAYER, which is what makes stored weights into inference rather than storage,
 *     following arXiv 2109.02157 (Plateau & Bellec, "Learning with Holographic Reduced
 *     Representations"): a dense layer is expressible as bind + bundle + similarity. Both the input
 *     and every weight row are encoded with the SAME positional key set, so an output row's score is
 *     the inner product of the input with that row — the layer's own arithmetic, performed by the
 *     hypervector algebra. `exactLinear` computes the same layer directly, so the coordinator's
 *     tests can measure the VSA path against the truth instead of asserting it.
 */
(function (root) {
  "use strict";

  /* ---------------------------------------------------------------- type tables
   * e = elements per block, b = bytes per block. F32/F16 have one element per "block".
   * These are llama.cpp's block sizes and must not be approximated: a wrong stride silently
   * misreads every value after the first block. */
  var TYPE_BLOCK = {
    0:  { e: 1,   b: 4 },    // F32
    1:  { e: 1,   b: 2 },    // F16
    2:  { e: 32,  b: 18 },   // Q4_0   d(2) qs(16)
    3:  { e: 32,  b: 20 },   // Q4_1   d(2) m(2) qs(16)
    6:  { e: 32,  b: 22 },   // Q5_0   d(2) qh(4) qs(16)
    7:  { e: 32,  b: 24 },   // Q5_1   d(2) m(2) qh(4) qs(16)
    8:  { e: 32,  b: 34 },   // Q8_0   d(2) qs(32)
    10: { e: 256, b: 84 },   // Q2_K   scales(16) qs(64) d(2) dmin(2)
    11: { e: 256, b: 110 },  // Q3_K   hmask(32) qs(64) scales(12) d(2)
    12: { e: 256, b: 144 },  // Q4_K   d(2) dmin(2) scales(12) qs(128)
    13: { e: 256, b: 176 },  // Q5_K   d(2) dmin(2) scales(12) qh(32) qs(128)
    14: { e: 256, b: 210 },  // Q6_K   ql(128) qh(64) scales(16) d(2)
    20: { e: 32,  b: 18 },   // IQ4_NL d(2) qs(16)
    30: { e: 1,   b: 2 }     // BF16   one element per block: the top 16 bits of a float32
  };

  var TYPE_NAME = { 0: "F32", 1: "F16", 2: "Q4_0", 3: "Q4_1", 6: "Q5_0", 7: "Q5_1",
                    8: "Q8_0", 10: "Q2_K", 11: "Q3_K", 12: "Q4_K", 13: "Q5_K",
                    14: "Q6_K", 20: "IQ4_NL", 30: "BF16" };

  /* IQ4_NL nibbles index a NON-LINEAR grid (llama.cpp kvalues_iq4nl), so the four-bit value is not
   * a multiplier. Note: the archive has two copies of this table and they DISAGREE in their last two
   * entries — gguf-core.js ends 69, 89, 113 while gguf-distill.js ends 69, 87, 108. This file uses
   * gguf-core.js's values (the file the toolkit's dequantTensor is verified against) and flags the
   * discrepancy rather than silently picking one. */
  var KVALUES_IQ4NL = [-127, -104, -83, -65, -49, -35, -22, -10,
                       1, 13, 25, 38, 53, 69, 89, 113];

  /* ---------------------------------------------------------------- helpers */

  function asBytes(buf) {
    if (buf instanceof Uint8Array) return buf;
    if (buf instanceof ArrayBuffer) return new Uint8Array(buf);
    if (buf && buf.buffer instanceof ArrayBuffer) {
      return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
    }
    throw new Error("expected an ArrayBuffer or Uint8Array");
  }

  /* IEEE 754 half -> single. Exponent bias 15, 10 mantissa bits. Subnormals are 2^-14 * m/1024;
   * exponent 31 is inf (m=0) or NaN, which must not be folded into a finite number. */
  function f16ToF32(h) {
    var s = (h & 0x8000) ? -1 : 1;
    var e = (h >> 10) & 0x1f;
    var m = h & 0x03ff;
    if (e === 0) return s * Math.pow(2, -14) * (m / 1024);
    if (e === 31) return m ? NaN : s * Infinity;
    return s * Math.pow(2, e - 15) * (1 + m / 1024);
  }

  /* bfloat16 -> float32. A bfloat16 is literally the top 16 bits of an IEEE-754 single, so the
   * conversion is a shift into the high half of a 32-bit word. Ported verbatim from
   * D:/JS-GGUF/GGUF-SURGERY/gguf-core.js, `bf16ToF32`. */
  function bf16ToF32(u16) {
    var b = new DataView(new ArrayBuffer(4));
    b.setUint32(0, (u16 << 16) >>> 0, true);
    return b.getFloat32(0, true);
  }

  function fmtBytes(n) {
    if (n === null || n === undefined || !isFinite(n)) return "?";
    var units = ["B", "KB", "MB", "GB", "TB"], i = 0, v = n;
    while (v >= 1024 && i < units.length - 1) { v /= 1024; i++; }
    return (i === 0 ? Math.round(v) : v.toFixed(2)) + " " + units[i];
  }

  /* A 64-bit integrity digest, dependency-free. The archive's Python uses SHA-256; a browser page
   * with no imports cannot, so this is an FNV-1a pair over two different primitives — explicitly NOT
   * cryptographic. The byte-exact round-trip test is the real guarantee; this catches corruption. */
  function digestFnv1a64(bytes) {
    var h1 = 0x811c9dc5 >>> 0, h2 = 0x84222325 >>> 0;
    for (var i = 0; i < bytes.length; i++) {
      var b = bytes[i];
      h1 = Math.imul(h1 ^ b, 0x01000193) >>> 0;
      h2 = Math.imul(h2 ^ b, 0x85ebca6b) >>> 0;
      h2 = (h2 ^ (h2 >>> 13)) >>> 0;
    }
    var hex = function (x) { return ("00000000" + x.toString(16)).slice(-8); };
    return hex(h1) + hex(h2);
  }

  /* ---------------------------------------------------------------- part 1: dequantise */

  /* Exact inverse of every quantisation llama.cpp writes for the supported types.
   * Returns a Float32Array of `count` values, or null for an unsupported type.
   * `count` may exceed what `bytes` holds (a prefix buffer): values past the available blocks stay 0
   * and the block count is clamped, so a short buffer can never read past its end. */
  function dequant(typeId, bytes, count) {
    var u8 = asBytes(bytes);
    var blk = TYPE_BLOCK[typeId];
    if (!blk) return null;
    var out = new Float32Array(count);
    var dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
    var f16 = function (o) { return f16ToF32(dv.getUint16(o, true)); };
    var b, base, i, j, idx, q, ql, qh, d, m;

    switch (typeId) {
      case 0: { // F32, direct
        var n0 = Math.min(count, Math.floor(u8.byteLength / 4));
        for (i = 0; i < n0; i++) out[i] = dv.getFloat32(i * 4, true);
        return out;
      }
      case 1: { // F16, direct
        var n1 = Math.min(count, Math.floor(u8.byteLength / 2));
        for (i = 0; i < n1; i++) out[i] = f16(i * 2);
        return out;
      }
      case 2: { // Q4_0: byte j holds element j in its LOW nibble and element j+16 in its HIGH
        var b2 = Math.min(Math.floor(u8.byteLength / 18), Math.ceil(count / 32));
        for (b = 0; b < b2; b++) {
          base = b * 18; d = f16(base);
          for (i = 0; i < 32; i++) {
            idx = b * 32 + i;
            if (idx < count) {
              q = (i < 16) ? (u8[base + 2 + i] & 0x0f) : (u8[base + 2 + (i - 16)] >> 4);
              out[idx] = d * (q - 8);      // 4-bit unsigned, zero-point 8
            }
          }
        }
        return out;
      }
      case 3: { // Q4_1: d, m, then the same nibble split; zero-point carried as a minimum
        var b3 = Math.min(Math.floor(u8.byteLength / 20), Math.ceil(count / 32));
        for (b = 0; b < b3; b++) {
          base = b * 20; d = f16(base); m = f16(base + 2);
          for (i = 0; i < 32; i++) {
            idx = b * 32 + i;
            if (idx < count) {
              q = (i < 16) ? (u8[base + 4 + i] & 0x0f) : (u8[base + 4 + (i - 16)] >> 4);
              out[idx] = d * q + m;
            }
          }
        }
        return out;
      }
      case 6: { // Q5_0: d, qh(4 bytes = the 5th bit of all 32 elements), qs(16)
        var b6 = Math.min(Math.floor(u8.byteLength / 22), Math.ceil(count / 32));
        for (b = 0; b < b6; b++) {
          base = b * 22; d = f16(base);
          var qh6 = (u8[base + 2] | (u8[base + 3] << 8) | (u8[base + 4] << 16) |
                     (u8[base + 5] << 24)) >>> 0;
          for (i = 0; i < 32; i++) {
            idx = b * 32 + i;
            if (idx < count) {
              ql = (i < 16) ? (u8[base + 6 + i] & 0x0f) : (u8[base + 6 + (i - 16)] >> 4);
              q = ql | (((qh6 >> i) & 1) << 4);      // 5-bit, zero-point 16
              out[idx] = d * (q - 16);
            }
          }
        }
        return out;
      }
      case 7: { // Q5_1: d, m, qh(4), qs(16)
        var b7 = Math.min(Math.floor(u8.byteLength / 24), Math.ceil(count / 32));
        for (b = 0; b < b7; b++) {
          base = b * 24; d = f16(base); m = f16(base + 2);
          var qh7 = (u8[base + 4] | (u8[base + 5] << 8) | (u8[base + 6] << 16) |
                     (u8[base + 7] << 24)) >>> 0;
          for (i = 0; i < 32; i++) {
            idx = b * 32 + i;
            if (idx < count) {
              ql = (i < 16) ? (u8[base + 8 + i] & 0x0f) : (u8[base + 8 + (i - 16)] >> 4);
              q = ql | (((qh7 >> i) & 1) << 4);
              out[idx] = d * q + m;
            }
          }
        }
        return out;
      }
      case 8: { // Q8_0: d, then 32 SIGNED 8-bit values — no nibble packing to get wrong
        var b8 = Math.min(Math.floor(u8.byteLength / 34), Math.ceil(count / 32));
        for (b = 0; b < b8; b++) {
          base = b * 34; d = f16(base);
          for (i = 0; i < 32; i++) {
            idx = b * 32 + i;
            if (idx < count) out[idx] = dv.getInt8(base + 2 + i) * d;
          }
        }
        return out;
      }
      case 10: { // Q2_K: scales(16) qs(64) d(2) dmin(2) — 256 elems/block, 16 sub-blocks of 16
        var b10 = Math.min(Math.floor(u8.byteLength / 84), Math.ceil(count / 256));
        for (b = 0; b < b10; b++) {
          base = b * 84; d = f16(base + 80);
          var dmin10 = f16(base + 82);
          for (var s10 = 0; s10 < 16; s10++) {
            var sc10 = u8[base + s10], dl10 = d * (sc10 & 0x0f), ml10 = dmin10 * (sc10 >> 4);
            var qb10 = base + 16 + (s10 >> 3) * 32 + (s10 & 1) * 16, sh10 = ((s10 >> 1) & 3) * 2;
            for (j = 0; j < 16; j++) {
              idx = b * 256 + s10 * 16 + j;
              if (idx < count) out[idx] = dl10 * ((u8[qb10 + j] >> sh10) & 3) - ml10;
            }
          }
        }
        return out;
      }
      case 11: { // Q3_K: hmask(32) qs(64) scales(12) d(2) — 16 sub-blocks of 16
        var b11 = Math.min(Math.floor(u8.byteLength / 110), Math.ceil(count / 256));
        for (b = 0; b < b11; b++) {
          base = b * 110; d = f16(base + 108);
          var lsc = [], hsc = [];
          for (i = 0; i < 16; i++) lsc.push((u8[base + 96 + (i & 7)] >> (((i >> 3) & 1) * 4)) & 0x0f);
          for (i = 0; i < 16; i++) hsc.push((u8[base + 104 + (i & 3)] >> (((i >> 2) & 3) * 2)) & 3);
          for (var s11 = 0; s11 < 16; s11++) {
            var dl11 = d * ((lsc[s11] | (hsc[s11] << 4)) - 32);
            var qb11 = base + 32 + (s11 >> 3) * 32 + (s11 & 1) * 16, qs11 = ((s11 >> 1) & 3) * 2;
            var hb11 = base + (s11 & 1) * 16, hs11 = (s11 >> 1) & 7;
            for (j = 0; j < 16; j++) {
              idx = b * 256 + s11 * 16 + j;
              if (idx < count) {
                var ql11 = (u8[qb11 + j] >> qs11) & 3;
                var qh11 = (u8[hb11 + j] >> hs11) & 1;
                out[idx] = dl11 * (ql11 - ((qh11 ^ 1) << 2));
              }
            }
          }
        }
        return out;
      }
      case 12: { // Q4_K: d(2) dmin(2) scales(12) qs(128) — 8 sub-blocks of 32
        var b12 = Math.min(Math.floor(u8.byteLength / 144), Math.ceil(count / 256));
        for (b = 0; b < b12; b++) {
          base = b * 144; d = f16(base);
          var dmin12 = f16(base + 2);
          var sc12 = [], mn12 = [];
          for (i = 0; i < 4; i++) {
            sc12.push(u8[base + 4 + i] & 0x3f);
            mn12.push(u8[base + 8 + i] & 0x3f);
          }
          for (i = 0; i < 4; i++) {
            sc12.push((u8[base + 12 + i] & 0x0f) | ((u8[base + 4 + i] >> 2) & 0x30));
            mn12.push((u8[base + 12 + i] >> 4) | ((u8[base + 8 + i] >> 2) & 0x30));
          }
          for (var k12 = 0; k12 < 8; k12++) {
            var dl12 = d * sc12[k12], ml12 = dmin12 * mn12[k12];
            var qb12 = base + 16 + (k12 >> 1) * 32, sh12 = (k12 & 1) * 4;
            for (i = 0; i < 32; i++) {
              idx = b * 256 + k12 * 32 + i;
              if (idx < count) out[idx] = dl12 * ((u8[qb12 + i] >> sh12) & 0x0f) - ml12;
            }
          }
        }
        return out;
      }
      case 13: { // Q5_K: d(2) dmin(2) scales(12) qh(32) qs(128) — 8 sub-blocks of 32
        var b13 = Math.min(Math.floor(u8.byteLength / 176), Math.ceil(count / 256));
        for (b = 0; b < b13; b++) {
          base = b * 176; d = f16(base);
          var dmin13 = f16(base + 2);
          var sc13 = [], mn13 = [];
          for (i = 0; i < 4; i++) {
            sc13.push(u8[base + 4 + i] & 0x3f);
            mn13.push(u8[base + 8 + i] & 0x3f);
          }
          for (i = 0; i < 4; i++) {
            sc13.push((u8[base + 12 + i] & 0x0f) | ((u8[base + 4 + i] >> 2) & 0x30));
            mn13.push((u8[base + 12 + i] >> 4) | ((u8[base + 8 + i] >> 2) & 0x30));
          }
          for (var k13 = 0; k13 < 8; k13++) {
            var dl13 = d * sc13[k13], ml13 = dmin13 * mn13[k13];
            var qb13 = base + 48 + (k13 >> 1) * 32, sh13 = (k13 & 1) * 4;
            for (i = 0; i < 32; i++) {
              idx = b * 256 + k13 * 32 + i;
              if (idx < count) {
                ql = (u8[qb13 + i] >> sh13) & 0x0f;
                qh = (u8[base + 16 + i] >> k13) & 1;
                out[idx] = dl13 * (ql | (qh << 4)) - ml13;
              }
            }
          }
        }
        return out;
      }
      case 14: { // Q6_K: ql(128) qh(64) scales(16 int8) d(f16 at 208); 16 sub-blocks of 16
        var b14 = Math.min(Math.floor(u8.byteLength / 210), Math.ceil(count / 256));
        for (b = 0; b < b14; b++) {
          base = b * 210; d = f16(base + 208);
          for (var s = 0; s < 16; s++) {
            var dl = d * dv.getInt8(base + 192 + s);
            var qlb = base + (s >> 3) * 64 + (s & 3) * 16, qlShift = ((s >> 2) & 1) * 4;
            var qhb = base + 128 + (s >> 3) * 32 + (s & 1) * 16, qhShift = ((s >> 1) & 3) * 2;
            for (j = 0; j < 16; j++) {
              idx = b * 256 + s * 16 + j;
              if (idx < count) {
                ql = (u8[qlb + j] >> qlShift) & 0x0f;
                qh = (u8[qhb + j] >> qhShift) & 0x03;
                out[idx] = dl * ((ql | (qh << 4)) - 32);   // 6-bit, zero-point 32
              }
            }
          }
        }
        return out;
      }
      case 20: { // IQ4_NL: d, 16 nibble bytes; low nibble -> element 2j, high -> 2j+1
        var b20 = Math.min(Math.floor(u8.byteLength / 18), Math.ceil(count / 32));
        for (b = 0; b < b20; b++) {
          base = b * 18; d = f16(base);
          for (j = 0; j < 16; j++) {
            q = u8[base + 2 + j];
            var i0 = b * 32 + 2 * j, i1 = i0 + 1;
            if (i0 < count) out[i0] = d * KVALUES_IQ4NL[q & 0x0f];
            if (i1 < count) out[i1] = d * KVALUES_IQ4NL[q >> 4];
          }
        }
        return out;
      }
      case 30: { // BF16: no block packing — one 16-bit word per element, direct
        var n30 = Math.min(count, Math.floor(u8.byteLength / 2));
        for (i = 0; i < n30; i++) out[i] = bf16ToF32(dv.getUint16(i * 2, true));
        return out;
      }
      default:
        return null;
    }
  }

  function supportedTypes() {
    var out = [];
    for (var k in TYPE_BLOCK) if (TYPE_BLOCK.hasOwnProperty(k)) out.push(Number(k));
    return out.sort(function (a, b) { return a - b; });
  }

  /* ---------------------------------------------------------------- tensor access */

  function findKV(header, key) {
    var kv = header.kv || [];
    for (var i = 0; i < kv.length; i++) if (kv[i].key === key) return kv[i].value;
    return undefined;
  }

  /* GGUF stores a tensor's offset relative to the data section, which llama.cpp pads up to
   * `general.alignment` (default 32). Getting this wrong shifts every value by up to 31 bytes and
   * produces plausible-looking garbage, so it is derived from the file's own KV rather than assumed.
   * A cheap way for a test to confirm it: a real F32 embedding tensor must dequantise to small
   * finite floats, not denormals or NaNs. */
  function alignment(header) {
    var a = findKV(header, "general.alignment");
    return (typeof a === "number" && a > 0) ? a : 32;
  }

  function dataOffset(header) {
    var a = alignment(header);
    return Math.ceil(header.headerBytes / a) * a;
  }

  function tensorByteLength(t) {
    var blk = TYPE_BLOCK[t.typeId];
    if (!blk) return null;
    var n = Number(t.params);
    return Math.ceil(n / blk.e) * blk.b;
  }

  function findTensor(header, name) {
    var ts = header.tensors || [];
    for (var i = 0; i < ts.length; i++) if (ts[i].name === name) return ts[i];
    return null;
  }

  /* Compact index of what a file holds — used by the UI to show which tensors are readable. */
  function listTensors(header, nameFilter) {
    var ts = header.tensors || [], out = [];
    for (var i = 0; i < ts.length; i++) {
      var t = ts[i];
      if (nameFilter && t.name.indexOf(nameFilter) === -1) continue;
      out.push({
        name: t.name,
        dims: t.dims.slice(),
        typeId: t.typeId,
        typeName: TYPE_NAME[t.typeId] || String(t.typeId),
        params: Number(t.params),
        byteLength: tensorByteLength(t),
        supported: !!TYPE_BLOCK[t.typeId]
      });
    }
    return out;
  }

  /* Read and dequantise one whole tensor. Returns null values when the type is unsupported (the
   * caller reports that) — never a partially-guessed matrix. */
  function readTensor(buf, header, name) {
    var t = findTensor(header, name);
    if (!t) return null;
    var len = tensorByteLength(t);
    var bytes = asBytes(buf);
    var start = dataOffset(header) + Number(t.offset);
    if (len === null) {
      return { name: name, typeId: t.typeId, typeName: TYPE_NAME[t.typeId] || String(t.typeId),
               dims: t.dims.slice(), count: Number(t.params), byteLength: null, values: null,
               supported: false, dataStart: start };
    }
    if (start + len > bytes.byteLength) {
      return { name: name, typeId: t.typeId, typeName: TYPE_NAME[t.typeId] || String(t.typeId),
               dims: t.dims.slice(), count: Number(t.params), byteLength: len, values: null,
               supported: !!TYPE_BLOCK[t.typeId], dataStart: start,
               error: "tensor extends past the buffer (need " + (start + len - bytes.byteLength) +
                      " more bytes)" };
    }
    var slice = bytes.subarray(start, start + len);
    var values = dequant(t.typeId, slice, Number(t.params));
    return { name: name, typeId: t.typeId, typeName: TYPE_NAME[t.typeId] || String(t.typeId),
             dims: t.dims.slice(), count: Number(t.params), byteLength: len, values: values,
             supported: !!TYPE_BLOCK[t.typeId], dataStart: start };
  }

  /* ---------------------------------------------------------------- part 2: lossless HDC memory */

  /* One dimension per stored bit: bit 1 -> +1, bit 0 -> -1, bit-packed so the vector occupies
   * exactly the same bytes as the source. This is the archive's scheme and it is EXACT: it is a
   * representation, not a compression, and the dimension grows with the data (N = bytes * 8). */
  function packBipolar(u8) {
    var bytes = asBytes(u8);
    return { bytes: bytes.slice(0), dimensionBits: bytes.length * 8 };
  }

  function unpackBipolar(packed, nBytes) {
    var src = asBytes(packed);
    if (nBytes === undefined || nBytes === null) nBytes = src.length;
    if (nBytes > src.length) throw new Error("unpackBipolar: " + nBytes + " bytes requested from " +
                                             src.length + " packed");
    return src.slice(0, nBytes);
  }

  /* The expanded form, for callers that want the literal +1/-1 vector (8x memory — which is exactly
   * why the stored form stays bit-packed). */
  function toBipolarVector(u8) {
    var bytes = asBytes(u8), out = new Int8Array(bytes.length * 8), n = 0, j, b;
    for (var i = 0; i < bytes.length; i++) {
      b = bytes[i];
      for (j = 0; j < 8; j++) out[n++] = (b & (1 << j)) ? 1 : -1;
    }
    return out;
  }

  function fromBipolarVector(vec) {
    var out = new Uint8Array(Math.ceil(vec.length / 8));
    for (var i = 0; i < vec.length; i++) if (vec[i] > 0) out[i >> 3] |= 1 << (i & 7);
    return out;
  }

  /* Page a tensor's exact bytes into independently verifiable HDC pages. */
  function storeLossless(buf, header, name, opts) {
    opts = opts || {};
    var pageBytes = opts.pageBytes || (1024 * 1024);
    var t = findTensor(header, name);
    if (!t) return null;
    var len = tensorByteLength(t);
    if (len === null) return { name: name, supported: false,
                               typeName: TYPE_NAME[t.typeId] || String(t.typeId) };
    var all = asBytes(buf);
    var start = dataOffset(header) + Number(t.offset);
    // Uint8Array.slice truncates silently, which would digest a short tensor as if it were whole
    if (start + len > all.byteLength) {
      return { name: name, supported: false,
               typeName: TYPE_NAME[t.typeId] || String(t.typeId),
               error: "tensor extends past the buffer (need " + (start + len - all.byteLength) +
                      " more bytes)" };
    }
    var tensorBytes = all.slice(start, start + len);
    var pages = [], pageIndex = 0;
    for (var off = 0; off < tensorBytes.length; off += pageBytes) {
      var chunk = tensorBytes.subarray(off, Math.min(off + pageBytes, tensorBytes.length));
      var packed = packBipolar(chunk);
      pages.push({ index: pageIndex++, byteLength: chunk.length,
                   dimensionBits: packed.dimensionBits,
                   digest: digestFnv1a64(chunk), bytes: packed.bytes });
    }
    return {
      name: name, typeName: TYPE_NAME[t.typeId] || String(t.typeId), dims: t.dims.slice(),
      supported: true, byteLength: tensorBytes.length, dimensionBits: tensorBytes.length * 8,
      pageBytes: pageBytes, pageCount: pages.length, pages: pages,
      digest: digestFnv1a64(tensorBytes),
      // the honest accounting: the dimension equals the data, so this saves nothing by itself
      exact: true, compressive: false,
      ggufBytes: len
    };
  }

  /* Rebuild the exact source bytes. Verifies each page digest and the whole-tensor digest, so a
   * corrupted or mis-sized store fails loudly instead of yielding wrong weights. */
  function restoreLossless(store) {
    var total = 0, i;
    for (i = 0; i < store.pages.length; i++) total += store.pages[i].byteLength;
    var out = new Uint8Array(total), at = 0;
    for (i = 0; i < store.pages.length; i++) {
      var p = store.pages[i];
      var chunk = unpackBipolar(p.bytes, p.byteLength);
      if (p.digest && digestFnv1a64(chunk) !== p.digest) {
        throw new Error("HDC page " + p.index + " failed its digest");
      }
      out.set(chunk, at); at += chunk.length;
    }
    if (store.digest && digestFnv1a64(out) !== store.digest) {
      throw new Error("restored tensor failed its digest");
    }
    return out;
  }

  /* ---------------------------------------------------------------- part 3: ternary (BitNet b1.58) */

  function meanAbs(values, from, to) {
    var s = 0, i;
    for (i = from; i < to; i++) s += Math.abs(values[i]);
    return (to > from) ? s / (to - from) : 0;
  }

  /* W = s * T, T in {-1,0,+1}, s = absmean (arXiv 2402.17764). Two bit-planes, so 2 bits per weight
   * plus one 4-byte scale per block. blockSize = 0 means a single scale for the whole array. */
  function toTernary(values, opts) {
    opts = opts || {};
    var blockSize = opts.blockSize | 0;
    var n = values.length;
    var pos = new Uint8Array((n + 7) >> 3), neg = new Uint8Array((n + 7) >> 3);
    var scales = [];
    var step = (blockSize > 0) ? blockSize : n;
    for (var s0 = 0; s0 < n; s0 += step) {
      var end = Math.min(n, s0 + step);
      var sc = meanAbs(values, s0, end);
      if (!(sc > 0)) sc = 1;   // an all-zero block would divide by zero; T is 0 there anyway
      scales.push(sc);
      for (var i = s0; i < end; i++) {
        var t = Math.round(values[i] / sc);
        if (t > 1) t = 1; else if (t < -1) t = -1;
        if (t === 1) pos[i >> 3] |= 1 << (i & 7);
        else if (t === -1) neg[i >> 3] |= 1 << (i & 7);
      }
    }
    return {
      pos: pos, neg: neg, count: n, blockSize: blockSize, scales: scales,
      scale: scales.length === 1 ? scales[0] : 0,
      // 2 packed bit-planes; the information content of one ternary symbol is log2(3)
      perValueBits: 2,
      entropyBits: Math.log(3) / Math.LN2,
      bytes: ternaryBytes(n, blockSize),
      zeros: countZeros(pos, neg, n)
    };
  }

  function countZeros(pos, neg, n) {
    var z = 0;
    for (var i = 0; i < n; i++) {
      var p = (pos[i >> 3] >> (i & 7)) & 1, m = (neg[i >> 3] >> (i & 7)) & 1;
      if (!p && !m) z++;
    }
    return z;
  }

  /* Exact storage cost: two bit-planes plus one 4-byte scale per block. */
  function ternaryBytes(n, blockSize) {
    var blocks = blockSize > 0 ? Math.ceil(n / blockSize) : 1;
    return 2 * Math.ceil(n / 8) + blocks * 4;
  }

  function fromTernary(t, blockSize) {
    var out = new Float32Array(t.count);
    var bs = blockSize === undefined || blockSize === null ? t.blockSize : blockSize;
    var step = (bs > 0) ? bs : t.count;
    for (var i = 0; i < t.count; i++) {
      var sc = (t.scales && t.scales.length > 1) ? t.scales[Math.floor(i / step)] : t.scale;
      if (sc === undefined) sc = (t.scales && t.scales.length) ? t.scales[0] : 1;
      var p = (t.pos[i >> 3] >> (i & 7)) & 1, m = (t.neg[i >> 3] >> (i & 7)) & 1;
      out[i] = p ? sc : (m ? -sc : 0);
    }
    return out;
  }

  /* Measured against the ORIGINAL values, so the page can report real error rather than a claim. */
  function ternaryError(values, blockSize) {
    var t = toTernary(values, { blockSize: blockSize });
    var rec = fromTernary(t, blockSize);
    var se = 0, fro = 0, froErr = 0, maxAbs = 0, i, d;
    for (i = 0; i < values.length; i++) {
      d = rec[i] - values[i];
      se += d * d;
      fro += values[i] * values[i];
      froErr += d * d;
      if (Math.abs(d) > maxAbs) maxAbs = Math.abs(d);
    }
    return {
      count: values.length, blockSize: blockSize, perValueBits: 2,
      rmse: Math.sqrt(se / values.length),
      maxAbs: maxAbs,
      // ||W - rec||_F / ||W||_F — a RELATIVE ERROR, never a reconstruction
      relativeFrobenius: fro > 0 ? Math.sqrt(froErr / fro) : 0,
      zerosFraction: t.zeros / values.length,
      bytes: t.bytes
    };
  }

  /* ---------------------------------------------------------------- part 4: the VSA linear layer */

  /* Positional keys. Deterministic from the VSA's own seed, so reloading reproduces the same layer
   * exactly — which is what lets the *stored* rows be rebuilt rather than persisted. */
  function makeKeys(vsa, n) {
    var keys = new Array(n);
    for (var i = 0; i < n; i++) keys[i] = vsa.vector("wkey:" + i);
    return keys;
  }

  function complement(v) {
    var o = new Uint8Array(v.length);
    for (var i = 0; i < v.length; i++) o[i] = (~v[i]) & 0xff;
    return o;
  }

  /* sign( sum_i w[i] * key[i] ), evaluated dimension-wise.
   *
   * This is exactly what vsa.bundle's weighted majority computes: at dimension d the bit is set iff
   * the weight summed over the keys whose bit d is SET is at least half the total weight, and
   * because total = S+ + S- that is precisely S+ >= S-, i.e. sign(sum_i w[i]*key_i[d]). Negative
   * weights are therefore fine.
   *
   * Two guards are needed. vsa.bundle returns an all-clear vector when the weights sum to <= 0
   * instead of the signs, so a negative-sum encoding is computed on negated weights and the result
   * complemented (negating the weights negates the vector exactly). And an exactly-zero sum makes
   * the half-threshold zero, which bundle's early return also short-circuits, so that single case is
   * broken by a 1e-9 tie-break on one weight — it can only change dimensions whose sum is within
   * 1e-9 of a tie. */
  function signBundle(vsa, w, keys) {
    var n = Math.min(w.length, keys.length);
    var sum = 0, i;
    for (i = 0; i < n; i++) sum += w[i];
    var list = keys.slice(0, n);
    var weights = new Float64Array(n);
    for (i = 0; i < n; i++) weights[i] = w[i];

    if (sum === 0) {
      if (n === 0) return new Uint8Array(vsa.D >>> 3);
      weights[n - 1] += 1e-9;
      return vsa.bundle(list, weights);
    }
    if (sum > 0) return vsa.bundle(list, weights);
    for (i = 0; i < n; i++) weights[i] = -weights[i];
    return complement(vsa.bundle(list, weights));
  }

  /* Encode a real-valued input vector as one hypervector. */
  function encodeVector(vsa, x, keys) { return signBundle(vsa, x, keys); }

  /* Encode one hypervector per output row of a flat row-major matrix, using the SAME keys. */
  function encodeRows(vsa, values, cols, keys) {
    var rows = Math.floor(values.length / cols);
    var out = new Array(rows), row = new Float64Array(cols);
    for (var r = 0; r < rows; r++) {
      for (var c = 0; c < cols; c++) row[c] = values[r * cols + c];
      out[r] = signBundle(vsa, row, keys);
    }
    return out;
  }

  /* The layer's own arithmetic, performed by the hypervector algebra. Scores are similarities in
   * [-1,1]: their RANKING corresponds to the linear layer's output ranking, which is what a decode
   * consumes. Magnitudes are not preserved, and nothing here claims they are. */
  function linearVSA(vsa, xhv, rowHvs) {
    var out = new Float32Array(rowHvs.length);
    for (var j = 0; j < rowHvs.length; j++) out[j] = vsa.similarity(xhv, rowHvs[j]);
    return out;
  }

  /* The ground truth to measure the above against: y_j = sum_i W[j][i] * x[i]. */
  function exactLinear(x, values, cols) {
    var rows = Math.floor(values.length / cols);
    var out = new Float32Array(rows);
    for (var r = 0; r < rows; r++) {
      var s = 0, off = r * cols;
      for (var c = 0; c < cols; c++) s += values[off + c] * x[c];
      out[r] = s;
    }
    return out;
  }

  /* Rank agreement between two score vectors — the honest way to compare the VSA layer with the
   * exact one. 1.0 means identical order. */
  function rankAgreement(a, b) {
    var ia = order(a), ib = order(b), same = 0, i;
    var n = Math.min(ia.length, ib.length);
    for (i = 0; i < n; i++) if (ia[i] === ib[i]) same++;
    return n ? same / n : 0;
  }

  /* Top-k agreement: the fraction of the k highest-ranked rows the two score vectors agree on.
   * This is the measurement that matters for a decode (which row wins); full-order agreement is
   * dominated by the tail and understates a layer that ranks its top candidates correctly. */
  function topKAgreement(a, b, k) {
    var ka = order(a).slice(0, k), kb = order(b).slice(0, k);
    var set = {}, hits = 0, i;
    for (i = 0; i < kb.length; i++) set[kb[i]] = true;
    for (i = 0; i < ka.length; i++) if (set[ka[i]]) hits++;
    return ka.length ? hits / ka.length : 0;
  }

  function argmax(v) {
    var best = -Infinity, bi = -1;
    for (var i = 0; i < v.length; i++) if (v[i] > best) { best = v[i]; bi = i; }
    return bi;
  }

  function order(v) {
    var idx = new Array(v.length);
    for (var i = 0; i < v.length; i++) idx[i] = i;
    idx.sort(function (x, y) { return v[y] - v[x]; });
    return idx;
  }

  var api = {
    // part 1
    dequant: dequant, f16ToF32: f16ToF32, bf16ToF32: bf16ToF32, supportedTypes: supportedTypes,
    listTensors: listTensors, readTensor: readTensor, findTensor: findTensor,
    tensorByteLength: tensorByteLength, dataOffset: dataOffset, alignment: alignment,
    TYPE_BLOCK: TYPE_BLOCK, TYPE_NAME: TYPE_NAME, KVALUES_IQ4NL: KVALUES_IQ4NL,
    // part 2
    packBipolar: packBipolar, unpackBipolar: unpackBipolar,
    toBipolarVector: toBipolarVector, fromBipolarVector: fromBipolarVector,
    storeLossless: storeLossless, restoreLossless: restoreLossless,
    // part 3
    toTernary: toTernary, fromTernary: fromTernary, ternaryError: ternaryError,
    ternaryBytes: ternaryBytes,
    // part 4
    makeKeys: makeKeys, encodeVector: encodeVector, encodeRows: encodeRows,
    linearVSA: linearVSA, exactLinear: exactLinear, rankAgreement: rankAgreement,
    topKAgreement: topKAgreement, argmax: argmax, order: order,
    // misc
    fmtBytes: fmtBytes, digestFnv1a64: digestFnv1a64
  };

  if (typeof module !== "undefined" && module.exports) module.exports = api;
  root.HDCWeights = api;
})(typeof globalThis !== "undefined" ? globalThis : this);