DMCRA Lab: elastic level-set quantisation over real GGUF weights, measured end to end

gguf.js

@@ -0,0 +1,300 @@
+/* GGUF reader for the HDC engine — browser-native, no dependencies, no server.
+ *
+ * Same container spec as templates/static/gguf/gguf.js (GGUF v1/v2/v3; v1 uses 32-bit KV
+ * string lengths, v2+ use 64-bit) and the same type tables, which mirror
+ * templates/gguf_util.py so both implementations agree on every file.
+ *
+ * What this copy adds, and why the HDC engine needs its own reader:
+ *
+ *   1. REAL VOCABULARIES. The metadata block carries tokenizer.ggml.tokens — the model's own
+ *      vocabulary (512 entries for stories260K, 151,936 for Qwen2.5-0.5B). The toolkit reader
+ *      truncates every array to 16 entries to bound memory, which is right for an inspector
+ *      and useless for generation. Here, arrays named in `fullArrayKeys` are kept in full.
+ *
+ *   2. BOUNDED PREFIXES. A page that streams a multi-GB model over HTTP only ever holds a
+ *      prefix of it, so EOF can arrive in the middle of the tensor index. That is not an
+ *      error: the metadata block precedes the index, so all KVs are already parsed and the
+ *      result is returned with truncated:true.
+ *
+ *   3. ONE DECODER. 151,936 short strings decoder-per-string is pathologically slow; a single
+ *      TextDecoder instance is reused for every string in the file.
+ */
+(function (root) {
+  "use strict";
+
+  const MAGIC = 0x46554747; // "GGUF" read as a little-endian u32 (0x47,0x47,0x55,0x46)
+
+  const GGML_TYPES = {
+    0: "F32", 1: "F16", 2: "Q4_0", 3: "Q4_1", 6: "Q5_0", 7: "Q5_1",
+    8: "Q8_0", 9: "Q8_1", 10: "Q2_K", 11: "Q3_K", 12: "Q4_K", 13: "Q5_K",
+    14: "Q6_K", 15: "Q8_K", 16: "IQ2_XXS", 17: "IQ2_XS", 18: "IQ3_XXS",
+    19: "IQ1_S", 20: "IQ4_NL", 21: "IQ3_S", 22: "IQ2_S", 23: "IQ4_XS",
+    24: "I8", 25: "I16", 26: "I32", 27: "I64", 28: "F64", 29: "IQ1_M",
+    30: "BF16", 34: "TQ1_0", 35: "TQ2_0", 39: "MXFP4"
+  };
+
+  const BITS = {
+    F32: 32, F16: 16, BF16: 16, F64: 64,
+    "Q8_0": 8.5, "Q8_1": 8.5, I8: 8, I16: 16, I32: 32, I64: 64,
+    "Q6_K": 6.5625, "Q5_K": 5.5, "Q5_0": 5.5, "Q5_1": 6.0,
+    "Q4_K": 4.5, "Q4_0": 4.5, "Q4_1": 5.0, "IQ4_NL": 4.5, "IQ4_XS": 4.25,
+    "Q3_K": 3.4375, "IQ3_XXS": 3.06, "IQ3_S": 3.44,
+    "Q2_K": 2.5625, "IQ2_XXS": 2.06, "IQ2_XS": 2.31, "IQ2_S": 2.5,
+    "IQ1_S": 1.56, "IQ1_M": 1.75, "TQ1_0": 1.69, "TQ2_0": 2.06
+  };
+
+  const T_NAME = { 0: "u8", 1: "i8", 2: "u16", 3: "i16", 4: "u32", 5: "i32", 6: "f32",
+                   7: "bool", 8: "string", 9: "array", 10: "u64", 11: "i64", 12: "f64" };
+
+  // Arrays these keys name are the ones generation actually consumes: keep them whole.
+  const DEFAULT_FULL = ["tokenizer.ggml.tokens", "tokenizer.ggml.merges",
+                        "tokenizer.ggml.token_type", "tokenizer.ggml.scores"];
+
+  const MAX_ARRAY_KEEP = 16;   // bound memory on unrelated big arrays, as the toolkit does
+  const MAX_FULL_ITEMS = 400000;
+
+  // tokenizer.ggml.token_type values (llama.cpp)
+  const TOKEN_TYPE = { 1: "NORMAL", 2: "UNKNOWN", 3: "CONTROL", 4: "USER_DEFINED",
+                       5: "UNUSED", 6: "BYTE" };
+
+  const MARKER = /^\u2026?\.{3} (\d+) items total$/;
+
+  class GgufError extends Error {}
+
+  class Reader {
+    constructor(buffer, opts) {
+      this.view = new DataView(buffer);
+      this.bytes = new Uint8Array(buffer);
+      this.pos = 0;
+      this.sizeBytes = buffer.byteLength;
+      this.decoder = typeof TextDecoder !== "undefined" ? new TextDecoder("utf-8") : null;
+      this.maxArrayItems = opts.maxArrayItems;
+      this.maxFullItems = opts.maxFullItems;
+      // smallest legal header: magic 4 + version 4 + tensor_count 8 + kv_count 8
+      if (buffer.byteLength < 24) throw new GgufError("buffer too small for a GGUF header");
+      this.magic = this.view.getUint32(0, true);
+      if (this.magic !== MAGIC) throw new GgufError("not a GGUF file (bad magic)");
+      this.pos = 4;
+      this.version = this.u32();
+      if (this.version !== 1 && this.version !== 2 && this.version !== 3) {
+        throw new GgufError("unsupported GGUF version " + this.version);
+      }
+      this.nTensors = this.u64();
+      this.nKv = this.u64();
+    }
+    need(n) {
+      if (this.pos + n > this.sizeBytes) throw new GgufError("unexpected end of file");
+    }
+    u8() { this.need(1); return this.view.getUint8(this.pos++); }
+    i8() { this.need(1); return this.view.getInt8(this.pos++); }
+    u16() { this.need(2); const v = this.view.getUint16(this.pos, true); this.pos += 2; return v; }
+    i16() { this.need(2); const v = this.view.getInt16(this.pos, true); this.pos += 2; return v; }
+    u32() { this.need(4); const v = this.view.getUint32(this.pos, true); this.pos += 4; return v; }
+    i32() { this.need(4); const v = this.view.getInt32(this.pos, true); this.pos += 4; return v; }
+    f32() { this.need(4); const v = this.view.getFloat32(this.pos, true); this.pos += 4; return v; }
+    f64() { this.need(8); const v = this.view.getFloat64(this.pos, true); this.pos += 8; return v; }
+    u64() {
+      this.need(8);
+      const v = this.view.getBigUint64(this.pos, true);
+      this.pos += 8;
+      return v <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(v) : v.toString();
+    }
+    i64() {
+      this.need(8);
+      const v = this.view.getBigInt64(this.pos, true);
+      this.pos += 8;
+      return v <= BigInt(Number.MAX_SAFE_INTEGER) && v >= BigInt(Number.MIN_SAFE_INTEGER)
+        ? Number(v) : v.toString();
+    }
+    bool() { return this.u8() !== 0; }
+    str() {
+      const len = this.version === 1 ? this.u32() : this.u64();
+      const n = Number(len);
+      this.need(n);
+      const at = this.pos;
+      this.pos += n;
+      const slice = this.bytes.subarray(at, at + n);
+      if (this.decoder) return this.decoder.decode(slice);
+      let s = "";
+      for (let i = 0; i < slice.length; i++) s += String.fromCharCode(slice[i]);
+      return s;
+    }
+    /* mode: "keep"  -> at most maxArrayItems entries, then a "... N items total" marker
+     *       "full"  -> every entry (up to maxFullItems), for the tokenizer arrays
+     *       "drop"  -> nested array inside an array: consume it, keep only a marker */
+    value(type, mode) {
+      switch (type) {
+        case 0: return this.u8();
+        case 1: return this.i8();
+        case 2: return this.u16();
+        case 3: return this.i16();
+        case 4: return this.u32();
+        case 5: return this.i32();
+        case 6: return this.f32();
+        case 7: return this.bool();
+        case 8: return this.str();
+        case 9: {
+          const elem = this.u32();
+          const count = Number(this.u64());
+          if (mode === "drop") {
+            for (let i = 0; i < count; i++) this.value(elem, "drop");
+            return "<" + count + " items>";
+          }
+          const limit = Math.min(count, mode === "full" ? this.maxFullItems : this.maxArrayItems);
+          const vals = new Array(limit);
+          for (let i = 0; i < count; i++) {
+            const v = this.value(elem, "drop");
+            if (i < limit) vals[i] = v;
+          }
+          if (count > limit) vals.push("... " + count + " items total");
+          return vals;
+        }
+        case 10: return this.u64();
+        case 11: return this.i64();
+        case 12: return this.f64();
+        default: throw new GgufError("unknown metadata type " + type);
+      }
+    }
+  }
+
+  function wantFull(key, fullArrayKeys) {
+    for (let i = 0; i < fullArrayKeys.length; i++) {
+      const k = fullArrayKeys[i];
+      if (k === key) return true;
+      if (k.charAt(k.length - 1) === "*" && key.indexOf(k.slice(0, -1)) === 0) return true;
+    }
+    return false;
+  }
+
+  function parse(buffer, opts) {
+    opts = opts || {};
+    const maxArrayItems = opts.maxArrayItems === undefined ? MAX_ARRAY_KEEP : opts.maxArrayItems;
+    const maxFullItems = opts.maxFullItems === undefined ? MAX_FULL_ITEMS : opts.maxFullItems;
+    const fullArrayKeys = opts.fullArrayKeys || DEFAULT_FULL;
+    const metadataOnly = !!opts.metadataOnly;
+
+    const r = new Reader(buffer, { maxArrayItems: maxArrayItems, maxFullItems: maxFullItems });
+
+    // The metadata block comes first, so a prefix stream parses it completely. A truncation
+    // *inside* the KV block is genuinely unusable and is allowed to throw.
+    const kv = [];
+    let declared = {};
+    for (let i = 0; i < r.nKv; i++) {
+      const key = r.str();
+      const type = r.u32();
+      const mode = wantFull(key, fullArrayKeys) ? "full" : "keep";
+      const value = r.value(type, mode);
+      kv.push({ key: key, type: T_NAME[type] || String(type), value: value });
+      if (Array.isArray(value)) {
+        const m = typeof value[value.length - 1] === "string" ? MARKER.exec(value[value.length - 1]) : null;
+        if (m) declared[key] = Number(m[1]);
+      }
+    }
+
+    const tensors = [];
+    let truncated = false;
+    if (!metadataOnly) {
+      for (let i = 0; i < r.nTensors; i++) {
+        try {
+          const name = r.str();
+          const nd = r.u32();
+          const dims = [];
+          for (let d = 0; d < nd; d++) dims.push(r.u64());
+          const typeId = r.u32();
+          const offset = r.u64();
+          let params = 1;
+          for (let j = 0; j < dims.length; j++) params *= Number(dims[j]);
+          tensors.push({ name: name, dims: dims, type: GGML_TYPES[typeId] || String(typeId),
+                         typeId: typeId, offset: offset, params: params });
+        } catch (e) {
+          if (!(e instanceof GgufError)) throw e;
+          truncated = true;   // a prefix buffer ends mid-index; the KVs above are already good
+          break;
+        }
+      }
+    }
+
+    return { version: r.version, nTensors: r.nTensors, nKv: r.nKv, kv: kv, tensors: tensors,
+             sizeBytes: r.sizeBytes, truncated: truncated, headerBytes: r.pos,
+             declaredArraySizes: declared };
+  }
+
+  /* Pull a string/number array out of a KV list, reporting the declared size when the reader
+   * had to truncate it so the caller can tell "512 tokens" from "first 16 of 151,936". */
+  function arrayOf(kvValue) {
+    if (!Array.isArray(kvValue)) return { items: [], declared: 0, truncated: false };
+    const last = kvValue.length ? kvValue[kvValue.length - 1] : null;
+    const m = typeof last === "string" ? MARKER.exec(last) : null;
+    if (m) return { items: kvValue.slice(0, kvValue.length - 1), declared: Number(m[1]), truncated: true };
+    return { items: kvValue, declared: kvValue.length, truncated: false };
+  }
+
+  function vocab(header) {
+    const d = {};
+    for (let i = 0; i < header.kv.length; i++) d[header.kv[i].key] = header.kv[i].value;
+    const toks = arrayOf(d["tokenizer.ggml.tokens"]);
+    const merg = arrayOf(d["tokenizer.ggml.merges"]);
+    const ttyp = arrayOf(d["tokenizer.ggml.token_type"]);
+    const scor = arrayOf(d["tokenizer.ggml.scores"]);
+    const num = function (k) { const v = d[k]; return typeof v === "number" ? v : null; };
+    const str = function (k) { const v = d[k]; return typeof v === "string" ? v : null; };
+    return {
+      tokens: toks.items,
+      merges: merg.items,
+      tokenTypes: ttyp.items,
+      scores: scor.items,
+      model: str("tokenizer.ggml.model"),
+      pre: str("tokenizer.ggml.pre"),
+      architecture: str("general.architecture"),
+      name: str("general.name"),
+      bosId: num("tokenizer.ggml.bos_token_id"),
+      eosId: num("tokenizer.ggml.eos_token_id"),
+      unkId: num("tokenizer.ggml.unknown_token_id"),
+      padId: num("tokenizer.ggml.padding_token_id"),
+      addBos: d["tokenizer.ggml.add_bos_token"] === true,
+      vocabSize: toks.items.length,
+      declaredVocabSize: toks.declared,
+      mergeCount: merg.items.length,
+      declaredMergeCount: merg.declared,
+      truncated: toks.truncated || merg.truncated
+    };
+  }
+
+  /* Human-readable token kinds for a handful of ids — used by the UI to show that the
+   * vocabulary really came from the model (BPE pieces, byte fallbacks, control tokens). */
+  function tokenKind(v, id) {
+    const t = v && v.tokenTypes ? v.tokenTypes[id] : undefined;
+    return TOKEN_TYPE[t] || (typeof t === "number" ? String(t) : "?");
+  }
+
+  function quantHistogram(tensors) {
+    const h = {};
+    for (let i = 0; i < tensors.length; i++) h[tensors[i].type] = (h[tensors[i].type] || 0) + 1;
+    return Object.entries(h).sort(function (a, b) { return b[1] - a[1]; });
+  }
+
+  function fmtBytes(n) {
+    if (n === null || n === undefined) return "?";
+    const units = ["B", "KB", "MB", "GB", "TB"];
+    let i = 0, v = n;
+    while (v >= 1024 && i < units.length - 1) { v /= 1024; i++; }
+    return (i === 0 ? v.toFixed(0) : v.toFixed(2)) + " " + units[i];
+  }
+
+  function estimateSizes(nParams, overhead) {
+    overhead = overhead || 0;
+    return Object.entries(BITS)
+      .map(function (e) {
+        return { quant: e[0], bits: e[1], bytes: Math.round(nParams * e[1] / 8 * (1 + overhead)) };
+      })
+      .sort(function (a, b) { return a.bits - b.bits; });
+  }
+
+  const api = { parse: parse, vocab: vocab, arrayOf: arrayOf, tokenKind: tokenKind,
+                quantHistogram: quantHistogram, fmtBytes: fmtBytes, estimateSizes: estimateSizes,
+                GGML_TYPES: GGML_TYPES, BITS: BITS, T_NAME: T_NAME, TOKEN_TYPE: TOKEN_TYPE,
+                DEFAULT_FULL: DEFAULT_FULL, GgufError: GgufError };
+
+  if (typeof module !== "undefined" && module.exports) module.exports = api;
+  root.HDCGGUF = api;
+})(typeof globalThis !== "undefined" ? globalThis : this);