Delete embed_hessian.py

embed_hessian.py

@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-"""Build a REAL full Hessian for layer-0 attention straight from a GGUF.
-No calibration data, no forward pass, no transformers.
-
-WHY THIS WORKS
-    The input to blk.0's attention projections is RMSNorm(token_embd[t]) for
-    whatever token t is at that position. So the input distribution is just
-    the (normalised) rows of the embedding matrix, weighted by token
-    frequency. Both are already in the GGUF. Assuming a uniform token prior:
-
-        H0 = (1/V) * sum_t  xt xt^T ,   xt = RMSNorm(token_embd[t]) * gamma
-
-    where gamma is blk.0.attn_norm.weight if present. That is an exact
-    Hessian for that layer, not an approximation of one -- and with V ~ 128k
-    rows against n_in ~ 2-6k it is ~20-50x oversampled, far past the 2x rank
-    floor where estimated Hessians start being useful.
-
-SCOPE
-    Only blk.0.attn_q / attn_k / attn_v share this input space. Every other
-    tensor in the model takes input from a preceding layer's output, which
-    cannot be derived from weights alone. So this covers 3 tensors -- enough
-    to test whether the full-H path delivers over the diag-H path before
-    building a real calibration pipeline for the rest.
-
-USAGE
-    python3 embed_hessian.py model-f16.gguf h0.hxh
-    python3 hexstate_requantize.py in.gguf out.gguf --hessian h0.hxh \\
-        --imatrix imat.gguf --analog-imatrix
-
-    (Keep --analog-imatrix so the other 795 tensors still get diag-H;
-     --hessian entries take priority where they exist.)
-"""
-import sys, os
-import numpy as np
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from hexstate_requantize import (
-    write_hessian, QK_K, GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16,
-    f16_to_f32, bf16_to_f32,
-)
-from dump_tensor import load
-
-DAMP = 0.01
-EPS = 1e-5          # RMSNorm epsilon; llama-family default
-
-
-def fetch(path, infos, data_start, name):
-    ti = next((t for t in infos if t['name'] == name), None)
-    if ti is None:
-        return None, None
-    with open(path, 'rb') as f:
-        f.seek(data_start + ti['offset'])
-        raw = f.read(ti['data_size'])
-    if ti['type'] == GGML_TYPE_F32:
-        a = np.frombuffer(raw, dtype=np.float32).copy()
-    elif ti['type'] == GGML_TYPE_F16:
-        a = f16_to_f32(raw, ti['n_elements'])
-    elif ti['type'] == GGML_TYPE_BF16:
-        a = bf16_to_f32(raw, ti['n_elements'])
-    else:
-        raise SystemExit(f"{name}: type {ti['type']} is quantized; "
-                         f"use the F16/BF16 source GGUF")
-    return a[:ti['n_elements']], ti['dims']
-
-
-def main():
-    if len(sys.argv) < 3:
-        print(__doc__)
-        sys.exit(1)
-    gguf, out = sys.argv[1], sys.argv[2]
-    infos, data_start = load(gguf)
-
-    emb, dims = fetch(gguf, infos, data_start, 'token_embd.weight')
-    if emb is None:
-        raise SystemExit("no token_embd.weight in that GGUF")
-    n_in = dims[0]                      # ne0 = embedding dim = attention n_in
-    vocab = emb.size // n_in
-    print(f"  token_embd.weight: dims={dims} -> n_in={n_in}, vocab={vocab}")
-
-    if n_in % QK_K:
-        raise SystemExit(f"n_in={n_in} is not a multiple of {QK_K}; "
-                         f"the analog path cannot use it")
-
-    E = emb.reshape(vocab, n_in).astype(np.float64)
-
-    # RMSNorm each embedding row, exactly as the model does before attention
-    rms = np.sqrt((E * E).mean(axis=1, keepdims=True) + EPS)
-    X = E / rms
-
-    gamma, _ = fetch(gguf, infos, data_start, 'blk.0.attn_norm.weight')
-    if gamma is not None and gamma.size == n_in:
-        X *= gamma.astype(np.float64)[None, :]
-        print("  applied blk.0.attn_norm.weight")
-    else:
-        print("  WARNING: blk.0.attn_norm.weight not found; H is unscaled")
-
-    # H = X^T X / vocab, accumulated in chunks to bound peak memory
-    H = np.zeros((n_in, n_in), dtype=np.float64)
-    step = max(1, 2 ** 24 // n_in)
-    for i in range(0, vocab, step):
-        blk = X[i:i + step]
-        H += blk.T @ blk
-        print(f"\r    {min(i + step, vocab)}/{vocab} rows", end='', flush=True)
-    print()
-    H /= float(vocab)
-    H[np.diag_indices_from(H)] += DAMP * np.trace(H) / n_in
-
-    ratio = vocab / n_in
-    print(f"  oversampling: {ratio:.1f}x n_in "
-          f"({'ample' if ratio >= 2 else 'BELOW the 2x floor'})")
-    if ratio < 2:
-        raise SystemExit("  vocab < 2*n_in -- this H would be rank-poor. Stop.")
-
-    H32 = H.astype(np.float32)
-    entries = {}
-    for kind in ('attn_q', 'attn_k', 'attn_v'):
-        name = f'blk.0.{kind}.weight'
-        ti = next((t for t in infos if t['name'] == name), None)
-        if ti is None:
-            print(f"    skip {name}: not in this GGUF")
-            continue
-        if ti['dims'][0] != n_in:
-            print(f"    skip {name}: dims[0]={ti['dims'][0]} != {n_in}")
-            continue
-        entries[name] = H32
-        print(f"    {name}: full H, n_in={n_in}")
-
-    if not entries:
-        raise SystemExit("  no eligible layer-0 attention tensors found")
-
-    write_hessian(out, entries)
-    print(f"\n  wrote {out}: {len(entries)} tensors, "
-          f"{n_in * n_in * 4 / 2**20:.1f} MiB")
-    print(f"\n  run:  python3 hexstate_requantize.py in.gguf out.gguf \\")
-    print(f"            --hessian {out} --imatrix imat.gguf --analog-imatrix")
-    print(f"\n  then compare the three blk.0.attn_q/k/v tensors against your")
-    print(f"  diag-H run. Those three take full-H; everything else is diag-H,")
-    print(f"  so a whole-model perplexity delta will be small -- the signal is")
-    print(f"  in whether those tensors improve, not the aggregate.")
-
-
-if __name__ == '__main__':
-    main()