import os import torch import logging from vllm import LLM from vllm import SamplingParams from huggingface_hub import login import argparse class qwen7b: def __init__(self): parser = argparse.ArgumentParser(description='qwen7b Chat Interface') parser.add_argument('--temperature', '-t', type=float, default=0.2, help='Sampling temperature') parser.add_argument('--top_p', '-tp', type=float, default=0.9, help='Top p sampling parameter') parser.add_argument('--max_new_tokens', '-mt', type=int, default=1024, help='Maximum new tokens to generate') parser.add_argument('--max_length', '-ml', type=int, default=4096, help='Maximum total sequence length') parser.add_argument('--num_return_sequences', '-n', type=int, default=1, help='Number of response sequences to generate') # parser.add_argument('--peft', '-peft', type=bool, default=False, help='Whether to use PEFT') # parser.add_argument('--lora_model_path', '-lmp', type=str, default=None, help='Path to the LoRA model') self.args = parser.parse_args() torch.cuda.set_device(0) self.qwen = LLM( model="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", dtype=torch.bfloat16, # gpu_memory_utilization=0.8, trust_remote_code=True, quantization="bitsandbytes", load_format="bitsandbytes", # max_model_len=95000 ) def chat(self, messages): # Convert messages to the format expected by LLM params = SamplingParams(temperature=self.args.temperature, top_p=self.args.top_p, max_tokens=self.args.max_new_tokens, n=self.args.num_return_sequences) # Generate text using the model outputs = self.qwen.chat(messages=messages, sampling_params=params) output_text = outputs[0].outputs[0].text if "" in output_text: output_text_changed = output_text.split("")[1] # Extract and return the generated text return output_text_changed else: return output_text def topic_classifier(self, messages): # Only consider the last two user messages for topic classification # Add a system prompt to instruct the model to classify the topic system_prompt = { "role": "system", "content": ( "You are a helpful assistant that classifies the topic of the user's conversation. " "Given the last two messages from the user, identify the main topic being discussed. " "Respond with a concise topic label (e.g., 'Travel', 'Machine Learning', 'Healthcare', etc.)." ) } # Format messages with system prompt first formatted_messages = [system_prompt] + messages # Prepare sampling parameters params = SamplingParams( temperature=self.args.temperature, top_p=self.args.top_p, max_tokens=self.args.max_new_tokens, n=self.args.num_return_sequences ) # Generate text using the model outputs = self.qwen.chat(messages=formatted_messages, sampling_params=params) output_text = outputs[0].outputs[0].text if "" in output_text: output_text_changed = output_text.split("")[1] # Extract and return the generated text return output_text_changed else: return "temp" def query_generator(self,messages): # Only consider the last two user messages for topic classification # Add a system prompt to instruct the model to classify the topic system_prompt = { "role": "system", "content": ( "You are an expert visual storyteller and prompt engineer for image generation models like Midjourney, DALLĀ·E, and Stable Diffusion. " "Based on the last two user messages in a conversation, generate a detailed and vivid image generation prompt that describes a visually rich scene or concept. " "The prompt should include: clear subject(s), visual setting or background, style (e.g., realistic, digital art, cyberpunk, anime, oil painting), lighting and mood (e.g., warm sunset, eerie moonlight), " "and optional camera angle or lens effect (e.g., wide-angle, macro, aerial view). " "Ensure the result is coherent, imaginative, and optimized to generate high-quality images. " "Respond only with the final image prompt as a single line of text, with no explanations or formatting." ) } # Format messages with system prompt first formatted_messages = [system_prompt] + messages # Prepare sampling parameters params = SamplingParams( temperature=self.args.temperature, top_p=self.args.top_p, max_tokens=self.args.max_new_tokens, n=self.args.num_return_sequences ) outputs = self.qwen.chat(messages=formatted_messages, sampling_params=params) # Generate text using the model output_text = outputs[0].outputs[0].text if "" in output_text: output_text_changed = output_text.split("")[1] # Extract and return the generated text return output_text_changed else: return output_text if __name__ == "__main__": qwen7b_instance = qwen7b() messages = [{"role": "user", "content": "Hello how are you? what are you doing nowadays?"},{"role": "user", "content": "Hello how are you? what are you doing nowadays?"}] sampling_params = SamplingParams(temperature=0.2, top_p=0.9, max_tokens=1024, n=1) response = qwen7b_instance.topic_classifier(messages) print(response)