File size: 607 Bytes
c09efba
a5551e0
c09efba
 
 
 
 
 
 
 
a5551e0
c09efba
 
 
 
a5551e0
c09efba
a5551e0
c09efba
a5551e0
c09efba
a5551e0
c09efba
a5551e0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM

model_name = "Qwen/Qwen2.5-Coder-7B-Instruct"

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    device_map="auto",
    torch_dtype="auto",
    trust_remote_code=True
)

def chat(message, history):
    inputs = tokenizer(message, return_tensors="pt")

    output = model.generate(**inputs, max_new_tokens=300)

    reply = tokenizer.decode(output[0], skip_special_tokens=True)

    return reply

gr.ChatInterface(fn=chat).launch()