lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py

import os
import time

from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer
from hqq.engine.hf import HQQModelForCausalLM


def create_mxq_model(model_id, quant_config, config_id, load_quantized, save_dir):
    quantized = False
    model_file_size = 0
    quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
    if load_quantized and os.path.exists(quant_path):
        model = HQQModelForCausalLM.from_quantized(quant_path)
        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
        quantized = True
        model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
    else:
        model = HQQModelForCausalLM.from_pretrained(model_id)
        tokenizer = hggAutoTokenizer.from_pretrained(model_id)
    return model, tokenizer, quantized, model_file_size


def quantize_mxq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
    model_file_size = 0
    t1 = time.time()
    model.quantize_model(quant_config=quant_config)
    t2 = time.time()
    print("Took " + str(t2 - t1) + " seconds to quantize the model with MXQ")
    quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
    model.save_quantized(quant_path)
    # persistent the quantized model
    os.sync()
    model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
    return model, t2 - t1, model_file_size