Another 15% or so

dump_tensor.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Dump one tensor from a source GGUF (F16/BF16/F32) as raw little-endian
+float32, for feeding to ./bound.
+
+Usage:
+    python3 dump_tensor.py model-f16.gguf blk.2.attn_output.weight out.f32
+    python3 dump_tensor.py model-f16.gguf --list
+"""
+import struct, sys, os
+import numpy as np
+
+# Reuse the parsing helpers from the requantizer so the layout stays in sync.
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from hexstate_requantize import (
+    GGUF_MAGIC, GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16,
+    TYPE_BLOCK_SIZE, TYPE_BLOCK_BYTES,
+    read_string, read_kv_value, align_offset, bf16_to_f32, f16_to_f32,
+)
+
+
+def load(path):
+    with open(path, 'rb') as f:
+        assert struct.unpack('<I', f.read(4))[0] == GGUF_MAGIC, "bad GGUF magic"
+        struct.unpack('<I', f.read(4))[0]                    # version
+        n_tensors = struct.unpack('<Q', f.read(8))[0]
+        n_kv = struct.unpack('<Q', f.read(8))[0]
+        for _ in range(n_kv):
+            read_string(f)
+            read_kv_value(f, struct.unpack('<I', f.read(4))[0])
+
+        infos = []
+        for _ in range(n_tensors):
+            name = read_string(f)
+            n_dims = struct.unpack('<I', f.read(4))[0]
+            dims = [struct.unpack('<Q', f.read(8))[0] for _ in range(n_dims)]
+            ttype = struct.unpack('<I', f.read(4))[0]
+            offset = struct.unpack('<Q', f.read(8))[0]
+            n_el = 1
+            for d in dims:
+                n_el *= d
+            bs = TYPE_BLOCK_SIZE.get(ttype, 1)
+            bb = TYPE_BLOCK_BYTES.get(ttype, 4)
+            infos.append({'name': name, 'dims': dims, 'type': ttype,
+                          'offset': offset, 'n_elements': n_el,
+                          'data_size': ((n_el + bs - 1) // bs) * bb})
+        return infos, align_offset(f.tell())
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(__doc__)
+        sys.exit(1)
+    gguf = sys.argv[1]
+    infos, data_start = load(gguf)
+
+    if sys.argv[2] == '--list':
+        for ti in infos:
+            print(f"  {ti['name']:<50s} type={ti['type']:<3d} "
+                  f"dims={ti['dims']} n={ti['n_elements']:,}")
+        return
+
+    want = sys.argv[2]
+    out = sys.argv[3] if len(sys.argv) > 3 else 'tensor.f32'
+    ti = next((t for t in infos if t['name'] == want), None)
+    if ti is None:
+        print(f"no tensor named {want!r}; run with --list")
+        sys.exit(1)
+
+    with open(gguf, 'rb') as f:
+        f.seek(data_start + ti['offset'])
+        raw = f.read(ti['data_size'])
+
+    if ti['type'] == GGML_TYPE_F32:
+        a = np.frombuffer(raw, dtype=np.float32).copy()
+    elif ti['type'] == GGML_TYPE_F16:
+        a = f16_to_f32(raw, ti['n_elements'])
+    elif ti['type'] == GGML_TYPE_BF16:
+        a = bf16_to_f32(raw, ti['n_elements'])
+    else:
+        print(f"tensor is type {ti['type']} (already quantized) — "
+              f"dump from the F16/BF16 source GGUF instead")
+        sys.exit(1)
+
+    a = np.ascontiguousarray(a[:ti['n_elements']], dtype=np.float32)
+    a.tofile(out)
+    sigma = float(np.sqrt(np.mean(a.astype(np.float64) ** 2)))
+    v = a.astype(np.float64) ** 2
+    kurt = float(np.mean(v * v) / (np.mean(v) ** 2)) if np.mean(v) > 0 else 3.0
+    print(f"  wrote {out}: {a.size:,} elements, dims={ti['dims']}")
+    print(f"  sigma    = {sigma:.6e}")
+    print(f"  kurtosis = {kurt:.2f}   (gaussian = 3)")
+    print()
+    print(f"  next:  ./bound --file {out}")
+
+
+if __name__ == '__main__':
+    main()

embed_hessian.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Build a REAL full Hessian for layer-0 attention straight from a GGUF.
+No calibration data, no forward pass, no transformers.
+
+WHY THIS WORKS
+    The input to blk.0's attention projections is RMSNorm(token_embd[t]) for
+    whatever token t is at that position. So the input distribution is just
+    the (normalised) rows of the embedding matrix, weighted by token
+    frequency. Both are already in the GGUF. Assuming a uniform token prior:
+
+        H0 = (1/V) * sum_t  xt xt^T ,   xt = RMSNorm(token_embd[t]) * gamma
+
+    where gamma is blk.0.attn_norm.weight if present. That is an exact
+    Hessian for that layer, not an approximation of one -- and with V ~ 128k
+    rows against n_in ~ 2-6k it is ~20-50x oversampled, far past the 2x rank
+    floor where estimated Hessians start being useful.
+
+SCOPE
+    Only blk.0.attn_q / attn_k / attn_v share this input space. Every other
+    tensor in the model takes input from a preceding layer's output, which
+    cannot be derived from weights alone. So this covers 3 tensors -- enough
+    to test whether the full-H path delivers over the diag-H path before
+    building a real calibration pipeline for the rest.
+
+USAGE
+    python3 embed_hessian.py model-f16.gguf h0.hxh
+    python3 hexstate_requantize.py in.gguf out.gguf --hessian h0.hxh \\
+        --imatrix imat.gguf --analog-imatrix
+
+    (Keep --analog-imatrix so the other 795 tensors still get diag-H;
+     --hessian entries take priority where they exist.)
+"""
+import sys, os
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from hexstate_requantize import (
+    write_hessian, QK_K, GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16,
+    f16_to_f32, bf16_to_f32,
+)
+from dump_tensor import load
+
+DAMP = 0.01
+EPS = 1e-5          # RMSNorm epsilon; llama-family default
+
+
+def fetch(path, infos, data_start, name):
+    ti = next((t for t in infos if t['name'] == name), None)
+    if ti is None:
+        return None, None
+    with open(path, 'rb') as f:
+        f.seek(data_start + ti['offset'])
+        raw = f.read(ti['data_size'])
+    if ti['type'] == GGML_TYPE_F32:
+        a = np.frombuffer(raw, dtype=np.float32).copy()
+    elif ti['type'] == GGML_TYPE_F16:
+        a = f16_to_f32(raw, ti['n_elements'])
+    elif ti['type'] == GGML_TYPE_BF16:
+        a = bf16_to_f32(raw, ti['n_elements'])
+    else:
+        raise SystemExit(f"{name}: type {ti['type']} is quantized; "
+                         f"use the F16/BF16 source GGUF")
+    return a[:ti['n_elements']], ti['dims']
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(__doc__)
+        sys.exit(1)
+    gguf, out = sys.argv[1], sys.argv[2]
+    infos, data_start = load(gguf)
+
+    emb, dims = fetch(gguf, infos, data_start, 'token_embd.weight')
+    if emb is None:
+        raise SystemExit("no token_embd.weight in that GGUF")
+    n_in = dims[0]                      # ne0 = embedding dim = attention n_in
+    vocab = emb.size // n_in
+    print(f"  token_embd.weight: dims={dims} -> n_in={n_in}, vocab={vocab}")
+
+    if n_in % QK_K:
+        raise SystemExit(f"n_in={n_in} is not a multiple of {QK_K}; "
+                         f"the analog path cannot use it")
+
+    E = emb.reshape(vocab, n_in).astype(np.float64)
+
+    # RMSNorm each embedding row, exactly as the model does before attention
+    rms = np.sqrt((E * E).mean(axis=1, keepdims=True) + EPS)
+    X = E / rms
+
+    gamma, _ = fetch(gguf, infos, data_start, 'blk.0.attn_norm.weight')
+    if gamma is not None and gamma.size == n_in:
+        X *= gamma.astype(np.float64)[None, :]
+        print("  applied blk.0.attn_norm.weight")
+    else:
+        print("  WARNING: blk.0.attn_norm.weight not found; H is unscaled")
+
+    # H = X^T X / vocab, accumulated in chunks to bound peak memory
+    H = np.zeros((n_in, n_in), dtype=np.float64)
+    step = max(1, 2 ** 24 // n_in)
+    for i in range(0, vocab, step):
+        blk = X[i:i + step]
+        H += blk.T @ blk
+        print(f"\r    {min(i + step, vocab)}/{vocab} rows", end='', flush=True)
+    print()
+    H /= float(vocab)
+    H[np.diag_indices_from(H)] += DAMP * np.trace(H) / n_in
+
+    ratio = vocab / n_in
+    print(f"  oversampling: {ratio:.1f}x n_in "
+          f"({'ample' if ratio >= 2 else 'BELOW the 2x floor'})")
+    if ratio < 2:
+        raise SystemExit("  vocab < 2*n_in -- this H would be rank-poor. Stop.")
+
+    H32 = H.astype(np.float32)
+    entries = {}
+    for kind in ('attn_q', 'attn_k', 'attn_v'):
+        name = f'blk.0.{kind}.weight'
+        ti = next((t for t in infos if t['name'] == name), None)
+        if ti is None:
+            print(f"    skip {name}: not in this GGUF")
+            continue
+        if ti['dims'][0] != n_in:
+            print(f"    skip {name}: dims[0]={ti['dims'][0]} != {n_in}")
+            continue
+        entries[name] = H32
+        print(f"    {name}: full H, n_in={n_in}")
+
+    if not entries:
+        raise SystemExit("  no eligible layer-0 attention tensors found")
+
+    write_hessian(out, entries)
+    print(f"\n  wrote {out}: {len(entries)} tensors, "
+          f"{n_in * n_in * 4 / 2**20:.1f} MiB")
+    print(f"\n  run:  python3 hexstate_requantize.py in.gguf out.gguf \\")
+    print(f"            --hessian {out} --imatrix imat.gguf --analog-imatrix")
+    print(f"\n  then compare the three blk.0.attn_q/k/v tensors against your")
+    print(f"  diag-H run. Those three take full-H; everything else is diag-H,")
+    print(f"  so a whole-model perplexity delta will be small -- the signal is")
+    print(f"  in whether those tensors improve, not the aggregate.")
+
+
+if __name__ == '__main__':
+    main()