rayhan10 commited on
Commit
aaae1ab
·
verified ·
1 Parent(s): 925a39f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -15
app.py CHANGED
@@ -20,25 +20,19 @@ def to_text(c):
20
 
21
 
22
  @spaces.GPU(duration=120)
23
- def chat(message, history):
24
- msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history]
25
- msgs.append({"role": "user", "content": to_text(message)})
26
-
27
  ids = tokenizer.apply_chat_template(
28
  msgs, add_generation_prompt=True, return_tensors="pt",
29
  return_dict=True, enable_thinking=False,
30
  ).to("cuda")
 
 
 
31
 
32
- streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
33
- Thread(target=model.generate, kwargs=dict(
34
- **ids, streamer=streamer, max_new_tokens=1024,
35
- do_sample=True, temperature=0.7, top_p=0.95,
36
- )).start()
37
-
38
- text = ""
39
- for tok in streamer:
40
- text += tok
41
- yield text
42
 
 
 
 
43
 
44
- gr.ChatInterface(chat, title="Private Coding Assistant").launch()
 
20
 
21
 
22
  @spaces.GPU(duration=120)
23
+ def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
24
+ msgs = json.loads(messages_json)
 
 
25
  ids = tokenizer.apply_chat_template(
26
  msgs, add_generation_prompt=True, return_tensors="pt",
27
  return_dict=True, enable_thinking=False,
28
  ).to("cuda")
29
+ out = model.generate(**ids, max_new_tokens=int(max_tokens),
30
+ do_sample=True, temperature=float(temperature), top_p=0.95)
31
+ return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
32
 
 
 
 
 
 
 
 
 
 
 
33
 
34
+ with gr.Blocks() as demo:
35
+ gr.ChatInterface(chat, title="Private Coding Assistant")
36
+ gr.api(generate, api_name="generate")
37
 
38
+ demo.launch()