Spaces:
Sleeping
Sleeping
Download app.py from rayhan10/my-custom-coder: direct link, hf CLI and curl.
- Browser
- Download file 1.96 kB
-
https://huggingface.co/spaces/rayhan10/my-custom-coder/resolve/main/app.py
- Command line
-
hf download hf://spaces/rayhan10/my-custom-coder/app.py
-
curl -L -o app.py https://huggingface.co/spaces/rayhan10/my-custom-coder/resolve/main/app.py
1.96 kB
| import os, json | |
| from threading import Thread | |
| import gradio as gr | |
| import spaces | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer | |
| MODEL_ID = os.environ.get("MODEL_ID", "lukey03/Qwen3.5-9B-abliterated") | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) | |
| model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype=torch.bfloat16) | |
| model.to("cuda") | |
| def to_text(c): | |
| if isinstance(c, list): | |
| return "".join(p.get("text", "") for p in c if isinstance(p, dict)) | |
| return c or "" | |
| def chat(message, history): | |
| msgs = [{"role": h["role"], "content": to_text(h["content"])} for h in history] | |
| msgs.append({"role": "user", "content": to_text(message)}) | |
| ids = tokenizer.apply_chat_template( | |
| msgs, add_generation_prompt=True, return_tensors="pt", | |
| return_dict=True, enable_thinking=False, | |
| ).to("cuda") | |
| streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) | |
| Thread(target=model.generate, kwargs=dict( | |
| **ids, streamer=streamer, max_new_tokens=1024, | |
| do_sample=True, temperature=0.7, top_p=0.95, | |
| )).start() | |
| text = "" | |
| for tok in streamer: | |
| text += tok | |
| yield text | |
| def generate(messages_json: str, max_tokens: int = 1024, temperature: float = 0.7) -> str: | |
| msgs = json.loads(messages_json) | |
| ids = tokenizer.apply_chat_template( | |
| msgs, add_generation_prompt=True, return_tensors="pt", | |
| return_dict=True, enable_thinking=False, | |
| ).to("cuda") | |
| out = model.generate(**ids, max_new_tokens=int(max_tokens), | |
| do_sample=True, temperature=float(temperature), top_p=0.95) | |
| return tokenizer.decode(out[0][ids["input_ids"].shape[1]:], skip_special_tokens=True) | |
| with gr.Blocks() as demo: | |
| gr.ChatInterface(chat, title="Private Coding Assistant") | |
| gr.api(generate, api_name="generate") | |
| demo.launch() |