Spaces:
Sleeping
Sleeping
File size: 1,962 Bytes
61c4405 925a39f 946fb15 9efe6e8 925a39f 9efe6e8 925a39f 9efe6e8 925a39f 61c4405 925a39f 9efe6e8 925a39f 9efe6e8 61c4405 925a39f aaae1ab 925a39f aaae1ab 20946b4 aaae1ab 20946b4 aaae1ab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | import os, json
from threading import Thread
import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16)
model.to("cuda")
def to_text(c):
if isinstance(c, list):
return "".join(p.get("text", "") for p in c if isinstance(p, dict))
return c or ""
@spaces.GPU(duration=120)
def chat(message, history):
msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history]
msgs.append({"role": "user", "content": to_text(message)})
ids = tokenizer.apply_chat_template(
msgs, add_generation_prompt=True, return_tensors="pt",
return_dict=True, enable_thinking=False,
).to("cuda")
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
Thread(target=model.generate, kwargs=dict(
**ids, streamer=streamer, max_new_tokens=1024,
do_sample=True, temperature=0.7, top_p=0.95,
)).start()
text = ""
for tok in streamer:
text += tok
yield text
@spaces.GPU(duration=120)
def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
msgs = json.loads(messages_json)
ids = tokenizer.apply_chat_template(
msgs, add_generation_prompt=True, return_tensors="pt",
return_dict=True, enable_thinking=False,
).to("cuda")
out = model.generate(**ids, max_new_tokens=int(max_tokens),
do_sample=True, temperature=float(temperature), top_p=0.95)
return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
with gr.Blocks() as demo:
gr.ChatInterface(chat, title="Private Coding Assistant")
gr.api(generate, api_name="generate")
demo.launch() |