nlp-chapter6-nextgpt / scripts /verify_quant_linear.py
jp2023213625's picture
Upload folder using huggingface_hub
c1f7bd5 verified
Raw History Blame Contribute Delete
1.11 kB
"""Numerically check portable GPTQ unpacking + LoRA against a dense layer."""
import torch
from minimal_inference import GPTQLinear
torch.manual_seed(13)
qin=torch.randint(0,16,(32,16),dtype=torch.int32)
zero=torch.full((2,16),8,dtype=torch.int32)
groups=torch.arange(32,dtype=torch.int32)//16
# Non-monotonic grouping checks activation-order g_idx indexing.
groups=groups[torch.randperm(32)]
scales=torch.rand(2,16,dtype=torch.float16)
qw=torch.zeros(4,16,dtype=torch.int32)
qz=torch.zeros(2,2,dtype=torch.int32)
for j in range(8):
qw|=qin[j::8]<<(4*j)
qz|=(zero[:,j::8]-1)<<(4*j)
a=torch.randn(4,32,dtype=torch.float16)/20
b=torch.randn(16,4,dtype=torch.float16)/20
layer=GPTQLinear({'qweight':qw,'qzeros':qz,'scales':scales,'g_idx':groups},a,b)
x=torch.randn(1,3,32,device='cuda',dtype=torch.float16)
dense=((qin-zero[groups]).half()*scales[groups]).cuda()
expected=x@dense + (x@a.cuda().T)@b.cuda().T
with torch.no_grad():actual=layer(x)
torch.testing.assert_close(actual,expected,atol=.01,rtol=.01)
print('PASS: GPTQ 4bit unpacking, activation-order groups and LoRA match dense reference.')