| from transformers import AutoTokenizer, AutoModelForCausalLM |
| import gradio as gr |
| import torch |
|
|
| |
| model_name = "Qwen/Qwen-1_8B" |
|
|
| tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) |
| model = AutoModelForCausalLM.from_pretrained( |
| model_name, |
| device_map="auto", |
| torch_dtype=torch.float16, |
| trust_remote_code=True |
| ) |
|
|
| def respond(message): |
| inputs = tokenizer(message, return_tensors="pt").to(model.device) |
| outputs = model.generate(**inputs, max_new_tokens=150) |
| return tokenizer.decode(outputs[0], skip_special_tokens=True) |
|
|
| gr.Interface( |
| fn=respond, |
| inputs="text", |
| outputs="text", |
| title="Qwen Прокси для Janotaro.ai", |
| description="Работает на Qwen-1_8B" |
| ).launch() |
