import os, json from threading import Thread import gradio as gr import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated") tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16) model.to("cuda") def to_text(c): if isinstance(c, list): return "".join(p.get("text", "") for p in c if isinstance(p, dict)) return c or "" @spaces.GPU(duration=120) def chat(message, history): msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history] msgs.append({"role": "user", "content": to_text(message)}) ids = tokenizer.apply_chat_template( msgs, add_generation_prompt=True, return_tensors="pt", return_dict=True, enable_thinking=False, ).to("cuda") streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) Thread(target=model.generate, kwargs=dict( **ids, streamer=streamer, max_new_tokens=1024, do_sample=True, temperature=0.7, top_p=0.95, )).start() text = "" for tok in streamer: text += tok yield text @spaces.GPU(duration=120) def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str: msgs = json.loads(messages_json) ids = tokenizer.apply_chat_template( msgs, add_generation_prompt=True, return_tensors="pt", return_dict=True, enable_thinking=False, ).to("cuda") out = model.generate(**ids, max_new_tokens=int(max_tokens), do_sample=True, temperature=float(temperature), top_p=0.95) return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True) with gr.Blocks() as demo: gr.ChatInterface(chat, title="Private Coding Assistant") gr.api(generate, api_name="generate") demo.launch()