File size: 1,902 Bytes
c94ba5e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | import gradio as gr
import spaces
import torch
import os
from transformers import (
AutoTokenizer,
AutoModelForCausalLM,
BitsAndBytesConfig,
)
MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct"
HF_TOKEN = os.environ["HF_TOKEN"]
tokenizer = None
model = None
def load_model():
global tokenizer, model
if model is not None:
return
quant_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.float16,
bnb_4bit_use_double_quant=True,
)
tokenizer = AutoTokenizer.from_pretrained(
MODEL_ID,
token=HF_TOKEN,
)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
token=HF_TOKEN,
quantization_config=quant_config,
device_map="cuda",
torch_dtype=torch.float16,
)
print("Model loaded on:", model.device)
@spaces.GPU
def greet(message):
load_model()
messages = [
{
"role": "user",
"content": message,
}
]
prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
inputs = tokenizer(
prompt,
return_tensors="pt",
).to("cuda")
with torch.inference_mode():
outputs = model.generate(
**inputs,
max_new_tokens=256,
temperature=0.7,
top_p=0.9,
do_sample=True,
)
generated_tokens = outputs[0][inputs["input_ids"].shape[-1]:]
return tokenizer.decode(
generated_tokens,
skip_special_tokens=True,
)
demo = gr.Interface(
fn=greet,
inputs=gr.Textbox(
label="Message",
placeholder="Ask Llama something...",
),
outputs=gr.Textbox(
label="Llama 3.1 8B Response",
),
title="Llama 3.1 8B Instruct",
)
demo.launch() |