chatbot / app.py
ojoke's picture
Update app.py
d90cd18 verified
Raw History Blame Contribute Delete
1.93 kB
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
import torch
import threading
model_name = "Qwen/Qwen3-0.6B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
model_name,
torch_dtype="auto",
device_map="auto"
)
def predict(user_input):
messages = [
{"role": "user", "content": user_input}
]
text = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
enable_thinking=False
)
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
# generated_ids = model.generate(
# **model_inputs,
# max_new_tokens=32768
# )
# output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
# try:
# index = len(output_ids) - output_ids[::-1].index(151668)
# except ValueError:
# index = 0
# # thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
# content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
# return content
streamer = TextIteratorStreamer(
tokenizer, skip_prompt=True, skip_special_tokens=True
)
generation_kwargs = dict(
**model_inputs,
streamer=streamer,
max_new_tokens=32768
)
thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
thread.start()
for new_text in streamer:
yield new_text
with gr.Blocks() as demo:
gr.Markdown("# AI Chatbot")
with gr.Row(equal_height=True):
textbox = gr.Textbox(lines=1, show_label=False, placeholder="Prompt")
button = gr.Button("Generate", variant="primary")
output_box = gr.Textbox(label="AI Output", lines=10)
button.click(fn=predict, inputs=textbox, outputs=output_box)
demo.launch(share=True)