import torch from audiocraft.models import MusicGen from transformers import GPT2LMHeadModel, GPT2Tokenizer import pyttsx3 import gradio as gr from tempfile import NamedTemporaryFile import numpy as np import scipy.io.wavfile as wavfile from diffusers import StableDiffusionPipeline import matplotlib.pyplot as plt import librosa.display import librosa import soundfile as sf from PIL import Image import os # Set device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # MusicGen music_model = MusicGen.get_pretrained("small", device=device) # GPT-2 for conversation tokenizer = GPT2Tokenizer.from_pretrained("gpt2") gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) # Stable Diffusion for image generation pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) pipe = pipe.to(device) # Emotion detection for Text-to-Audio def get_emotion_tone(text): if any(word in text.lower() for word in ["happy", "joy", "excited"]): return "happy" elif any(word in text.lower() for word in ["sad", "down", "melancholy"]): return "sad" elif any(word in text.lower() for word in ["angry", "frustrated"]): return "angry" else: return "neutral" # Image generation using Stable Diffusion def generate_image(prompt, style="realistic"): styled_prompt = f"{style} style {prompt}" try: image = pipe(styled_prompt).images[0] temp_image = NamedTemporaryFile(delete=False, suffix=".png") image.save(temp_image.name) return temp_image.name except Exception as e: return f"Error generating image: {e}" # Convert Text to Audio with Emotion def text_to_audio(text): emotion = get_emotion_tone(text) engine = pyttsx3.init() engine.setProperty('rate', 150 if emotion == "neutral" else 180 if emotion == "happy" else 100 if emotion == "sad" else 200) engine.setProperty('volume', 0.8 if emotion == "neutral" else 1.0 if emotion == "happy" or emotion == "angry" else 0.5) temp_file = NamedTemporaryFile(delete=False, suffix=".mp3") engine.save_to_file(text, temp_file.name) engine.runAndWait() return temp_file.name # Music generation using MusicGen def generate_music(prompt): try: descriptions = [prompt] wav = music_model.generate(descriptions) temp_file = NamedTemporaryFile(delete=False, suffix=".wav") audio_data = wav.cpu().numpy() wavfile.write(temp_file.name, music_model.sample_rate, audio_data[0, 0]) return temp_file.name except Exception as e: return f"Error generating music: {e}" # Spectrogram generation from audio def generate_spectrogram(audio_path): try: y, sr = librosa.load(audio_path, sr=None) S = librosa.feature.melspectrogram(y, sr=sr) S_dB = librosa.power_to_db(S, ref=np.max) plt.figure(figsize=(10, 4)) librosa.display.specshow(S_dB, sr=sr, x_axis='time', y_axis='mel', cmap='coolwarm') plt.colorbar(format='%+2.0f dB') plt.title('Mel-frequency spectrogram') temp_image = NamedTemporaryFile(delete=False, suffix=".png") plt.savefig(temp_image.name) plt.close() return temp_image.name except Exception as e: return f"Error generating spectrogram: {e}" # Chat with AI (GPT-2) def chat_with_ai(user_input): try: inputs = tokenizer.encode(user_input, return_tensors="pt").to(device) outputs = gpt2_model.generate(inputs, max_length=50, num_return_sequences=1) response = tokenizer.decode(outputs[0], skip_special_tokens=True) return response except Exception as e: return f"Error in chat generation: {e}" # Simulate Video Generation using a Sequence of Images def generate_video(prompt): frames = [] for i in range(5): # Generate 5 frames as a sequence frame_prompt = f"{prompt} frame {i+1}" frame_path = generate_image(frame_prompt) frames.append(Image.open(frame_path)) temp_video = NamedTemporaryFile(delete=False, suffix=".gif") frames[0].save(temp_video.name, save_all=True, append_images=frames[1:], duration=500, loop=0) return temp_video.name # Main interface logic def main_interface(input_text, task_type, style): try: if task_type == "Conversation": response = chat_with_ai(input_text) image_path = generate_image(f"conversation about {input_text}", style) return response, None, image_path elif task_type == "Music": audio_path = generate_music(input_text) spectrogram_path = generate_spectrogram(audio_path) return "Music Generated", audio_path, spectrogram_path elif task_type == "Text to Audio": audio_path = text_to_audio(input_text) image_path = generate_image(f"text-to-audio conversion for {input_text}", style) return "Audio Generated", audio_path, image_path elif task_type == "Video Generation": video_path = generate_video(input_text) audio_path = generate_music(input_text) return "Video Generated", audio_path, video_path except Exception as e: return f"Error: {e}", None, None # Gradio interface setup interface = gr.Interface( fn=main_interface, inputs=[ gr.Textbox(label="Enter Text or Prompt"), gr.Radio(["Conversation", "Music", "Text to Audio", "Video Generation"], label="Select Task"), gr.Dropdown(["realistic", "abstract", "comic"], label="Select Style"), ], outputs=[ gr.Textbox(label="Generated Output"), gr.Audio(label="Generated Audio", type="filepath"), gr.Image(label="Generated Image", type="filepath"), ], live=False, ) interface.launch()