Story_Generation / deepseek.py
aiwithi's picture
End of training
28ab977 verified
Raw History Blame Contribute Delete
5.86 kB
import os
import torch
import logging
from vllm import LLM
from vllm import SamplingParams
from huggingface_hub import login
import argparse
class qwen7b:
def __init__(self):
parser = argparse.ArgumentParser(description='qwen7b Chat Interface')
parser.add_argument('--temperature', '-t', type=float, default=0.2, help='Sampling temperature')
parser.add_argument('--top_p', '-tp', type=float, default=0.9, help='Top p sampling parameter')
parser.add_argument('--max_new_tokens', '-mt', type=int, default=1024, help='Maximum new tokens to generate')
parser.add_argument('--max_length', '-ml', type=int, default=4096, help='Maximum total sequence length')
parser.add_argument('--num_return_sequences', '-n', type=int, default=1, help='Number of response sequences to generate')
# parser.add_argument('--peft', '-peft', type=bool, default=False, help='Whether to use PEFT')
# parser.add_argument('--lora_model_path', '-lmp', type=str, default=None, help='Path to the LoRA model')
self.args = parser.parse_args()
torch.cuda.set_device(0)
self.qwen = LLM(
model="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
dtype=torch.bfloat16,
# gpu_memory_utilization=0.8,
trust_remote_code=True,
quantization="bitsandbytes",
load_format="bitsandbytes",
# max_model_len=95000
)
def chat(self, messages):
# Convert messages to the format expected by LLM
params = SamplingParams(temperature=self.args.temperature, top_p=self.args.top_p, max_tokens=self.args.max_new_tokens, n=self.args.num_return_sequences)
# Generate text using the model
outputs = self.qwen.chat(messages=messages, sampling_params=params)
output_text = outputs[0].outputs[0].text
if "</think>" in output_text:
output_text_changed = output_text.split("</think>")[1]
# Extract and return the generated text
return output_text_changed
else:
return output_text
def topic_classifier(self, messages):
# Only consider the last two user messages for topic classification
# Add a system prompt to instruct the model to classify the topic
system_prompt = {
"role": "system",
"content": (
"You are a helpful assistant that classifies the topic of the user's conversation. "
"Given the last two messages from the user, identify the main topic being discussed. "
"Respond with a concise topic label (e.g., 'Travel', 'Machine Learning', 'Healthcare', etc.)."
)
}
# Format messages with system prompt first
formatted_messages = [system_prompt] + messages
# Prepare sampling parameters
params = SamplingParams(
temperature=self.args.temperature,
top_p=self.args.top_p,
max_tokens=self.args.max_new_tokens,
n=self.args.num_return_sequences
)
# Generate text using the model
outputs = self.qwen.chat(messages=formatted_messages, sampling_params=params)
output_text = outputs[0].outputs[0].text
if "</think>" in output_text:
output_text_changed = output_text.split("</think>")[1]
# Extract and return the generated text
return output_text_changed
else:
return "temp"
def query_generator(self,messages):
# Only consider the last two user messages for topic classification
# Add a system prompt to instruct the model to classify the topic
system_prompt = {
"role": "system",
"content": (
"You are an expert visual storyteller and prompt engineer for image generation models like Midjourney, DALL·E, and Stable Diffusion. "
"Based on the last two user messages in a conversation, generate a detailed and vivid image generation prompt that describes a visually rich scene or concept. "
"The prompt should include: clear subject(s), visual setting or background, style (e.g., realistic, digital art, cyberpunk, anime, oil painting), lighting and mood (e.g., warm sunset, eerie moonlight), "
"and optional camera angle or lens effect (e.g., wide-angle, macro, aerial view). "
"Ensure the result is coherent, imaginative, and optimized to generate high-quality images. "
"Respond only with the final image prompt as a single line of text, with no explanations or formatting."
)
}
# Format messages with system prompt first
formatted_messages = [system_prompt] + messages
# Prepare sampling parameters
params = SamplingParams(
temperature=self.args.temperature,
top_p=self.args.top_p,
max_tokens=self.args.max_new_tokens,
n=self.args.num_return_sequences
)
outputs = self.qwen.chat(messages=formatted_messages, sampling_params=params)
# Generate text using the model
output_text = outputs[0].outputs[0].text
if "</think>" in output_text:
output_text_changed = output_text.split("</think>")[1]
# Extract and return the generated text
return output_text_changed
else:
return output_text
if __name__ == "__main__":
qwen7b_instance = qwen7b()
messages = [{"role": "user", "content": "Hello how are you? what are you doing nowadays?"},{"role": "user", "content": "Hello how are you? what are you doing nowadays?"}]
sampling_params = SamplingParams(temperature=0.2, top_p=0.9, max_tokens=1024, n=1)
response = qwen7b_instance.topic_classifier(messages)
print(response)