| from transformers import AutoModelForCausalLM, AutoTokenizer |
| import spaces |
| import gradio as gr |
|
|
| model_name = "Qwen/Qwen2.5-7B-Instruct" |
|
|
| model = AutoModelForCausalLM.from_pretrained( |
| model_name, |
| torch_dtype="auto", |
| device_map="auto", |
| ) |
| tokenizer = AutoTokenizer.from_pretrained(model_name) |
|
|
| @spaces.GPU |
| def generate(prompt): |
| text = tokenizer.apply_chat_template( |
| prompt, |
| tokenize=False, |
| add_generation_prompt=True |
| ) |
| model_inputs = tokenizer([text], return_tensors="pt").to(model.device) |
| generated_ids = model.generate( |
| **model_inputs, |
| max_new_tokens=2048, |
| ) |
| generated_ids = [ |
| output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) |
| ] |
| response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] |
|
|
| return response |
|
|
|
|
| gr.Interface( |
| fn=generate, |
| inputs=gr.Text(), |
| outputs=gr.Text(), |
| ).launch() |
