my-custom-coder / app.py
rayhan10's picture
Update app.py
61c4405 verified
Raw History Blame Contribute Delete
1.96 kB
import os, json
from threading import Thread
import gradio as gr
import spaces
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16)
model.to("cuda")
def to_text(c):
if isinstance(c, list):
return "".join(p.get("text", "") for p in c if isinstance(p, dict))
return c or ""
@spaces.GPU(duration=120)
def chat(message, history):
msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history]
msgs.append({"role": "user", "content": to_text(message)})
ids = tokenizer.apply_chat_template(
msgs, add_generation_prompt=True, return_tensors="pt",
return_dict=True, enable_thinking=False,
).to("cuda")
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
Thread(target=model.generate, kwargs=dict(
**ids, streamer=streamer, max_new_tokens=1024,
do_sample=True, temperature=0.7, top_p=0.95,
)).start()
text = ""
for tok in streamer:
text += tok
yield text
@spaces.GPU(duration=120)
def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str:
msgs = json.loads(messages_json)
ids = tokenizer.apply_chat_template(
msgs, add_generation_prompt=True, return_tensors="pt",
return_dict=True, enable_thinking=False,
).to("cuda")
out = model.generate(**ids, max_new_tokens=int(max_tokens),
do_sample=True, temperature=float(temperature), top_p=0.95)
return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True)
with gr.Blocks() as demo:
gr.ChatInterface(chat, title="Private Coding Assistant")
gr.api(generate, api_name="generate")
demo.launch()