File size: 1,631 Bytes
64ddf8d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | import numpy as np
import pandas as pd
import torch
from hqq.core.quantize import Quantizer as hQuant
from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir
from lm_quant_toolkit.utils.safetensors import get_tensor
def quant_hqq(tensor, nbits, group_size=64, lp_norm=0.7, optimize=True):
opt_params = {
"lp_norm": lp_norm,
"iters": 100,
"early_stop": False,
}
wq, meta = hQuant.quantize(
tensor, nbits=nbits, group_size=group_size, optimize=optimize, **opt_params
)
return hQuant.dequantize(wq, meta)
if __name__ == "__main__":
model_id = "meta-llama/Llama-2-7b-hf"
base_dir = get_hf_model_storge_base_dir(model_id)
layer = 31
module = "self_attn.o_proj"
matrix_name = f"model.layers.{layer}.{module}.weight"
w = get_tensor(matrix_name, base_dir)
dick = []
for lp_norm in np.linspace(0.1, 0.9, 9):
for b in [3, 4, 8]:
for g in [32, 64, 128]:
wq_hqq = quant_hqq(w, nbits=b, lp_norm=lp_norm, group_size=g)
norm_hqq = torch.norm(w - wq_hqq)
dick.append(
{
"model": model_id.split("/")[1],
"layer": layer,
"module": module,
"lp_norm": lp_norm,
"b": b,
"g": g,
"fnorm": norm_hqq.item(),
}
)
print(f"FNorm HQQ(lp_norm={lp_norm:.2f}): {norm_hqq}")
df = pd.DataFrame(dick)
df.to_csv("lp_norm_tuning.csv", index=False)
|