test / app.py
sycn0410's picture
Update app.py
0f7c4e3 verified
Raw History Blame Contribute Delete
1.55 kB
import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
# 使用 Qwen3.5-9B 量化版本(显存占用约 6GB)
MODEL_ID = "Qwen/Qwen3.5-9B"
print("正在加载模型...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
device_map="auto",
trust_remote_code=True,
dtype=torch.float16,
)
print("模型加载完成!")
def generate(text, max_length=512):
if not text.strip():
return "请输入内容"
messages = [{"role": "user", "content": text}]
text_input = tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(text_input, return_tensors="pt").to(model.device)
outputs = model.generate(
**inputs,
max_new_tokens=max_length,
do_sample=True,
temperature=0.7,
pad_token_id=tokenizer.eos_token_id,
)
response = tokenizer.decode(
outputs[0][len(inputs.input_ids[0]):],
skip_special_tokens=True
)
return response
with gr.Blocks(title="Qwen 聊天", theme=gr.themes.Soft()) as demo:
gr.Markdown("# 🤖 Qwen3.5-9B 在线聊天")
input_text = gr.Textbox(label="输入", lines=3, placeholder="你好...")
output_text = gr.Textbox(label="回复", lines=6)
btn = gr.Button("发送", variant="primary")
btn.click(fn=generate, inputs=input_text, outputs=output_text)
if __name__ == "__main__":
demo.launch()