File size: 1,340 Bytes
64ddf8d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | import os
import time
from transformers import AutoModelForCausalLM, AutoTokenizer
from lm_quant_toolkit.adapter.common import get_model_storage_size
def create_bnb_model(model_id, quant_config, config_id, load_quantized, save_dir):
quantized = False
model_file_size = 0
quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
if load_quantized and os.path.exists(quant_path):
model = AutoModelForCausalLM.from_pretrained(quant_path)
tokenizer = AutoTokenizer.from_pretrained(model_id)
quantized = True
model_file_size = get_model_storage_size(quant_path)
else:
model = None
tokenizer = AutoTokenizer.from_pretrained(model_id)
return model, tokenizer, quantized, model_file_size
def quantize_bnb_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
model_file_size = 0
t1 = time.time()
model = AutoModelForCausalLM.from_pretrained(
model_id, quantization_config=quant_config
)
t2 = time.time()
print("Took " + str(t2 - t1) + " seconds to quantize the model with BnB")
quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
model.save_pretrained(quant_path)
# persistent the quantized model
os.sync()
model_file_size = get_model_storage_size(quant_path)
return model, t2 - t1, model_file_size
|