File size: 1,934 Bytes
7a0f47a
d90cd18
 
 
7a0f47a
b033f89
 
 
 
 
 
 
 
50801a2
101bfcf
 
b033f89
101bfcf
b033f89
 
 
 
d90cd18
b033f89
 
 
d90cd18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b033f89
d90cd18
b033f89
 
 
d90cd18
 
b033f89
d90cd18
 
50801a2
 
101bfcf
 
 
 
 
 
50801a2
b033f89
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import gradio as gr
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
import torch
import threading

model_name = "Qwen/Qwen3-0.6B"

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype="auto",
    device_map="auto"
)

def predict(user_input):
    messages = [
        {"role": "user", "content": user_input}
    ]
    text = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True,
        enable_thinking=False
    )
    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)

    # generated_ids = model.generate(
    #     **model_inputs,
    #     max_new_tokens=32768
    # )
    # output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() 

    # try:
    #     index = len(output_ids) - output_ids[::-1].index(151668)
    # except ValueError:
    #     index = 0

    # # thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
    # content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")

    # return content

    streamer = TextIteratorStreamer(
        tokenizer, skip_prompt=True, skip_special_tokens=True
    )

    generation_kwargs = dict(
        **model_inputs,
        streamer=streamer,
        max_new_tokens=32768
    )

    thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()

    for new_text in streamer:
        yield new_text

with gr.Blocks() as demo:
    gr.Markdown("# AI Chatbot")
    with gr.Row(equal_height=True):
        textbox = gr.Textbox(lines=1, show_label=False, placeholder="Prompt")
        button = gr.Button("Generate", variant="primary")
    output_box = gr.Textbox(label="AI Output", lines=10)
    button.click(fn=predict, inputs=textbox, outputs=output_box)

demo.launch(share=True)