import gradio as gr from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer import torch import threading model_name = "Qwen/Qwen3-0.6B" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForCausalLM.from_pretrained( model_name, torch_dtype="auto", device_map="auto" ) def predict(user_input): messages = [ {"role": "user", "content": user_input} ] text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False ) model_inputs = tokenizer([text], return_tensors="pt").to(model.device) # generated_ids = model.generate( # **model_inputs, # max_new_tokens=32768 # ) # output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() # try: # index = len(output_ids) - output_ids[::-1].index(151668) # except ValueError: # index = 0 # # thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n") # content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n") # return content streamer = TextIteratorStreamer( tokenizer, skip_prompt=True, skip_special_tokens=True ) generation_kwargs = dict( **model_inputs, streamer=streamer, max_new_tokens=32768 ) thread = threading.Thread(target=model.generate, kwargs=generation_kwargs) thread.start() for new_text in streamer: yield new_text with gr.Blocks() as demo: gr.Markdown("# AI Chatbot") with gr.Row(equal_height=True): textbox = gr.Textbox(lines=1, show_label=False, placeholder="Prompt") button = gr.Button("Generate", variant="primary") output_box = gr.Textbox(label="AI Output", lines=10) button.click(fn=predict, inputs=textbox, outputs=output_box) demo.launch(share=True)