lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py

import os
import time

from transformers import AutoModelForCausalLM, AutoTokenizer

from lm_quant_toolkit.adapter.common import get_model_storage_size


def create_bnb_model(model_id, quant_config, config_id, load_quantized, save_dir):
    quantized = False
    model_file_size = 0
    quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
    if load_quantized and os.path.exists(quant_path):
        model = AutoModelForCausalLM.from_pretrained(quant_path)
        tokenizer = AutoTokenizer.from_pretrained(model_id)
        quantized = True
        model_file_size = get_model_storage_size(quant_path)
    else:
        model = None
        tokenizer = AutoTokenizer.from_pretrained(model_id)
    return model, tokenizer, quantized, model_file_size


def quantize_bnb_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
    model_file_size = 0
    t1 = time.time()
    model = AutoModelForCausalLM.from_pretrained(
        model_id, quantization_config=quant_config
    )
    t2 = time.time()
    print("Took " + str(t2 - t1) + " seconds to quantize the model with BnB")
    quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
    model.save_pretrained(quant_path)
    # persistent the quantized model
    os.sync()
    model_file_size = get_model_storage_size(quant_path)
    return model, t2 - t1, model_file_size