Spaces:
Sleeping
Sleeping
File size: 4,287 Bytes
f3f48cd 5dd0555 f3f48cd 8952e7b 5dd0555 f3f48cd 5dd0555 8952e7b f3f48cd 5dd0555 f3f48cd 8952e7b 5dd0555 8952e7b 5dd0555 8952e7b 5dd0555 f3f48cd 8952e7b 5dd0555 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | """
Qwen3 Chat — Hugging Face Space
Deploy by pushing this folder to a Gradio Space on huggingface.co
"""
import os
import torch
import gradio as gr
from threading import Thread
from transformers import (
AutoModelForCausalLM,
AutoTokenizer,
TextIteratorStreamer,
BitsAndBytesConfig,
)
# ── Config ────────────────────────────────────────────────────────────────────
MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-4B")
MAX_NEW_TOK = int(os.getenv("MAX_NEW_TOKENS", "512"))
THINKING_MODE = os.getenv("THINKING_MODE", "false").lower() == "true"
USE_4BIT = os.getenv("USE_4BIT", "false").lower() == "true"
# ──────────────────────────────────────────────────────────────────────────────
print(f"Loading {MODEL_ID} | 4-bit={USE_4BIT} | thinking={THINKING_MODE}")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
load_kwargs = dict(device_map="auto")
if USE_4BIT:
load_kwargs["quantization_config"] = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_compute_dtype=torch.float16,
)
load_kwargs["torch_dtype"] = torch.float16
elif torch.cuda.is_available():
load_kwargs["torch_dtype"] = torch.float16
else:
load_kwargs["torch_dtype"] = torch.float32
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, **load_kwargs)
model.eval()
print("Model ready.")
def build_prompt(history: list[dict]) -> str:
"""Convert Gradio-style history to a tokenized prompt string."""
messages = [{"role": m["role"], "content": m["content"]} for m in history]
return tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
enable_thinking=THINKING_MODE,
)
def respond(message, history, system_prompt, max_tokens, temperature, top_p):
# Build message list
msgs = []
if system_prompt.strip():
msgs.append({"role": "system", "content": system_prompt.strip()})
for h in history:
msgs.append({"role": "user", "content": h[0]})
msgs.append({"role": "assistant", "content": h[1]})
msgs.append({"role": "user", "content": message})
prompt = build_prompt(msgs)
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(
tokenizer, skip_prompt=True, skip_special_tokens=True
)
gen_kwargs = dict(
**inputs,
streamer=streamer,
max_new_tokens=max_tokens,
do_sample=temperature > 0,
temperature=max(temperature, 1e-6),
top_p=top_p,
pad_token_id=tokenizer.eos_token_id,
)
thread = Thread(target=model.generate, kwargs=gen_kwargs)
thread.start()
partial = ""
for chunk in streamer:
partial += chunk
yield partial
# ── UI ────────────────────────────────────────────────────────────────────────
with gr.Blocks(title=f"Qwen3 Chat — {MODEL_ID}", theme=gr.themes.Soft()) as demo:
gr.Markdown(f"## 🤖 {MODEL_ID}\nStreaming chat powered by Qwen3 on Hugging Face Spaces.")
chatbot = gr.ChatInterface(
fn=respond,
additional_inputs=[
gr.Textbox(
label="System prompt",
value="You are a helpful assistant.",
lines=2,
),
gr.Slider(64, 2048, value=MAX_NEW_TOK, step=64, label="Max new tokens"),
gr.Slider(0.0, 2.0, value=0.7, step=0.05, label="Temperature"),
gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top-p"),
],
additional_inputs_accordion=gr.Accordion("⚙️ Settings", open=False),
examples=[
["Explain transformers in simple terms."],
["Write a haiku about machine learning."],
["What is the capital of France?"],
],
cache_examples=False,
)
demo.queue(max_size=5)
demo.launch() |