rayhan10 commited on
Commit
61c4405
·
verified ·
1 Parent(s): aaae1ab

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +24 -2
app.py CHANGED
@@ -1,4 +1,4 @@
1
- import os
2
  from threading import Thread
3
 
4
  import gradio as gr
@@ -9,7 +9,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStream
9
  MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated")
10
 
11
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
12
- model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16)
13
  model.to("cuda")
14
 
15
 
@@ -19,6 +19,28 @@ def to_text(c):
19
  return c or ""
20
 
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  @spaces.GPU(duration=120)
23
  def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
24
  msgs = json.loads(messages_json)
 
1
+ import os, json
2
  from threading import Thread
3
 
4
  import gradio as gr
 
9
  MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated")
10
 
11
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
12
+ model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16)
13
  model.to("cuda")
14
 
15
 
 
19
  return c or ""
20
 
21
 
22
+ @spaces.GPU(duration=120)
23
+ def chat(message, history):
24
+ msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history]
25
+ msgs.append({"role": "user", "content": to_text(message)})
26
+
27
+ ids = tokenizer.apply_chat_template(
28
+ msgs, add_generation_prompt=True, return_tensors="pt",
29
+ return_dict=True, enable_thinking=False,
30
+ ).to("cuda")
31
+
32
+ streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
33
+ Thread(target=model.generate, kwargs=dict(
34
+ **ids, streamer=streamer, max_new_tokens=1024,
35
+ do_sample=True, temperature=0.7, top_p=0.95,
36
+ )).start()
37
+
38
+ text = ""
39
+ for tok in streamer:
40
+ text += tok
41
+ yield text
42
+
43
+
44
  @spaces.GPU(duration=120)
45
  def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
46
  msgs = json.loads(messages_json)