File size: 1,962 Bytes
61c4405
925a39f
946fb15
9efe6e8
925a39f
 
 
9efe6e8
925a39f
9efe6e8
925a39f
61c4405
925a39f
9efe6e8
 
925a39f
 
 
 
9efe6e8
 
61c4405
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
925a39f
aaae1ab
 
925a39f
 
 
 
aaae1ab
 
 
20946b4
 
aaae1ab
 
 
20946b4
aaae1ab
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import os, json
from threading import Thread

import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer

MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated")

tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16)
model.to("cuda")


def to_text(c):
    if isinstance(c, list):
        return "".join(p.get("text", "") for p in c if isinstance(p, dict))
    return c or ""


@spaces.GPU(duration=120)
def chat(message, history):
    msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history]
    msgs.append({"role": "user", "content": to_text(message)})

    ids = tokenizer.apply_chat_template(
        msgs, add_generation_prompt=True, return_tensors="pt",
        return_dict=True, enable_thinking=False,
    ).to("cuda")

    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
    Thread(target=model.generate, kwargs=dict(
        **ids, streamer=streamer, max_new_tokens=1024,
        do_sample=True, temperature=0.7, top_p=0.95,
    )).start()

    text = ""
    for tok in streamer:
        text += tok
        yield text


@spaces.GPU(duration=120)
def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
    msgs = json.loads(messages_json)
    ids = tokenizer.apply_chat_template(
        msgs, add_generation_prompt=True, return_tensors="pt",
        return_dict=True, enable_thinking=False,
    ).to("cuda")
    out = model.generate(**ids, max_new_tokens=int(max_tokens),
                         do_sample=True, temperature=float(temperature), top_p=0.95)
    return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)


with gr.Blocks() as demo:
    gr.ChatInterface(chat, title="Private Coding Assistant")
    gr.api(generate, api_name="generate")

demo.launch()