ojoke commited on
Commit
d90cd18
·
verified ·
1 Parent(s): 9a335a0

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +30 -12
app.py CHANGED
@@ -1,5 +1,7 @@
1
  import gradio as gr
2
- from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
3
 
4
  model_name = "Qwen/Qwen3-0.6B"
5
 
@@ -18,25 +20,41 @@ def predict(user_input):
18
  messages,
19
  tokenize=False,
20
  add_generation_prompt=True,
21
- enable_thinking=True
22
  )
23
  model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
24
 
25
- generated_ids = model.generate(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  **model_inputs,
 
27
  max_new_tokens=32768
28
  )
29
- output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
30
-
31
- try:
32
- index = len(output_ids) - output_ids[::-1].index(151668)
33
- except ValueError:
34
- index = 0
35
 
36
- # thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
37
- content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
38
 
39
- return content
 
40
 
41
  with gr.Blocks() as demo:
42
  gr.Markdown("# AI Chatbot")
 
1
  import gradio as gr
2
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
3
+ import torch
4
+ import threading
5
 
6
  model_name = "Qwen/Qwen3-0.6B"
7
 
 
20
  messages,
21
  tokenize=False,
22
  add_generation_prompt=True,
23
+ enable_thinking=False
24
  )
25
  model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
26
 
27
+ # generated_ids = model.generate(
28
+ # **model_inputs,
29
+ # max_new_tokens=32768
30
+ # )
31
+ # output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
32
+
33
+ # try:
34
+ # index = len(output_ids) - output_ids[::-1].index(151668)
35
+ # except ValueError:
36
+ # index = 0
37
+
38
+ # # thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
39
+ # content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
40
+
41
+ # return content
42
+
43
+ streamer = TextIteratorStreamer(
44
+ tokenizer, skip_prompt=True, skip_special_tokens=True
45
+ )
46
+
47
+ generation_kwargs = dict(
48
  **model_inputs,
49
+ streamer=streamer,
50
  max_new_tokens=32768
51
  )
 
 
 
 
 
 
52
 
53
+ thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
54
+ thread.start()
55
 
56
+ for new_text in streamer:
57
+ yield new_text
58
 
59
  with gr.Blocks() as demo:
60
  gr.Markdown("# AI Chatbot")