#!/usr/bin/env python3 """ Deforum Music-to-Video Interface - Fixed for Gradio 4.44.1 """ import math import collections import re import gradio as gr import os import sys import json import traceback import subprocess import importlib.util from pathlib import Path import tempfile import zipfile from typing import Optional, Dict, Any, List, Tuple, Union import warnings warnings.filterwarnings("ignore") # Import dependencies with proper error handling print("Loading dependencies...") try: import numpy as np print("✅ numpy loaded") except ImportError: print("❌ numpy not found") sys.exit(1) try: import librosa HAS_LIBROSA = True print("✅ librosa loaded") except ImportError: HAS_LIBROSA = False print("❌ librosa not found - please install: pip install librosa soundfile") try: import whisper HAS_WHISPER = True print("✅ whisper loaded") except ImportError: HAS_WHISPER = False print("⚠️ whisper not found - transcription disabled") try: import openai HAS_OPENAI = True print("✅ openai loaded") except ImportError: HAS_OPENAI = False print("⚠️ openai not found - GPT expansion disabled") try: import requests print("✅ requests loaded") except ImportError: print("❌ requests not found") class DeforumMusicGenerator: def __init__(self): self.PRESET_VARIANTS = { "Calm": { "zoom_amt": 0.012, "rot_amt": 0.35, "trans_amt": 0.06, "denoise_base": 0.30, "beats_per_section": 32 }, "Punchy": { "zoom_amt": 0.024, "rot_amt": 0.9, "trans_amt": 0.15, "denoise_base": 0.36, "beats_per_section": 16 }, "Glitch": { "zoom_amt": 0.03, "rot_amt": 1.2, "trans_amt": 0.22, "denoise_base": 0.42, "beats_per_section": 8 } } def analyze_audio_file(self, audio_path: str, fps: int = 24) -> Dict[str, Any]: """Analyze audio file and extract musical features""" if not HAS_LIBROSA: raise RuntimeError( "librosa is required for audio analysis. Install with: pip install librosa soundfile") try: # Load audio file y, sr = librosa.load(audio_path, sr=None, mono=True) duration = float(librosa.get_duration(y=y, sr=sr)) # Extract audio features hop_length = 512 # RMS energy (volume/intensity) rms = librosa.feature.rms( y=y, frame_length=2048, hop_length=hop_length).flatten() # Spectral centroid (brightness) spectral_centroid = librosa.feature.spectral_centroid( y=y, sr=sr, hop_length=hop_length).flatten() # Beat tracking tempo, beat_frames = librosa.beat.beat_track( y=y, sr=sr, hop_length=hop_length) beat_times = librosa.frames_to_time( beat_frames, sr=sr, hop_length=hop_length) # Convert to video frame numbers total_frames = int(duration * fps) beat_frame_numbers = np.round(beat_times * fps).astype(int) # Interpolate audio features to video timeline audio_times = librosa.frames_to_time( np.arange(len(rms)), sr=sr, hop_length=hop_length) video_times = np.arange(total_frames) / fps # Interpolate and normalize features rms_video = np.interp(video_times, audio_times, rms) brightness_video = np.interp( video_times, audio_times, spectral_centroid) # Normalize to 0-1 range rms_normalized = self._normalize_array(rms_video) brightness_normalized = self._normalize_array(brightness_video) # Apply smoothing rms_smooth = self._smooth_array(rms_normalized) brightness_smooth = self._smooth_array(brightness_normalized) return { "duration": duration, "fps": fps, "total_frames": total_frames, "tempo": float(tempo), "beat_frames": beat_frame_numbers, "rms_smooth": rms_smooth, "brightness_smooth": brightness_smooth, "sr": sr } except Exception as e: raise RuntimeError(f"Audio analysis failed: {str(e)}") def _normalize_array(self, arr: np.ndarray) -> np.ndarray: """Normalize array to 0-1 range""" arr = np.asarray(arr, dtype=float) min_val, max_val = arr.min(), arr.max() if max_val - min_val < 1e-8: return np.zeros_like(arr) return (arr - min_val) / (max_val - min_val) def _smooth_array(self, arr: np.ndarray, window: int = 8) -> np.ndarray: """Apply moving average smoothing""" if window <= 1: return arr kernel = np.ones(window) / window return np.convolve(arr, kernel, mode='same') def generate_keyframes(self, analysis: Dict[str, Any], variant_settings: Dict[str, float]) -> Dict[str, str]: """Generate camera movement keyframes based on audio analysis""" beat_frames = analysis["beat_frames"] rms = analysis["rms_smooth"] total_frames = analysis["total_frames"] # Initialize keyframe lists zoom_keyframes = [] angle_keyframes = [] tx_keyframes = [] ty_keyframes = [] strength_keyframes = [] # Generate keyframes at beat positions for frame in beat_frames: if frame >= total_frames: continue # Get audio energy at this frame energy = rms[min(frame, len(rms) - 1)] # Calculate movement values based on energy zoom_factor = 1.0 + (energy - 0.5) * 2.0 * \ variant_settings["zoom_amt"] rotation = (energy - 0.5) * 2.0 * variant_settings["rot_amt"] translation = (energy - 0.5) * 2.0 * variant_settings["trans_amt"] # Strength (denoising) - higher energy = lower strength for more # variation strength = max( 0.1, min(0.9, variant_settings["denoise_base"] + (0.5 - energy) * 0.25)) # Add keyframes zoom_keyframes.append((frame, zoom_factor)) angle_keyframes.append((frame, rotation)) tx_keyframes.append((frame, translation)) # Slightly different for Y ty_keyframes.append((frame, translation * 0.7)) strength_keyframes.append((frame, strength)) # Convert to Deforum keyframe format return { "zoom": self._keyframes_to_string(zoom_keyframes), "angle": self._keyframes_to_string(angle_keyframes), "translation_x": self._keyframes_to_string(tx_keyframes), "translation_y": self._keyframes_to_string(ty_keyframes), "strength_schedule": self._keyframes_to_string( strength_keyframes, precision=3)} def _keyframes_to_string( self, keyframes: List[Tuple[int, float]], precision: int = 4) -> str: """Convert keyframe list to Deforum string format""" if not keyframes: return "0:(1.0)" if precision == 4 else "0:(0.65)" sorted_kf = sorted(keyframes, key=lambda x: x[0]) return ", ".join( f"{frame}:({value:.{precision}f})" for frame, value in sorted_kf) def generate_prompts(self, analysis: Dict[str, Any], base_prompt: str, style: str, theme: str, beats_per_section: int = 16) -> Dict[int, str]: """Generate animation prompts based on audio sections""" beat_frames = analysis["beat_frames"] rms = analysis["rms_smooth"] tempo = analysis["tempo"] total_frames = analysis["total_frames"] prompts = {} if len(beat_frames) < beats_per_section: # Short audio - single prompt energy = np.mean(rms) mood = self._generate_mood_description(energy, tempo) prompt_parts = [base_prompt] if theme: prompt_parts.append(theme) prompt_parts.append(mood) if style: prompt_parts.append(style) prompts[0] = ", ".join(prompt_parts) else: # Longer audio - section-based prompts for i in range(0, len(beat_frames), beats_per_section): section_beats = beat_frames[i:i + beats_per_section] if section_beats is None or len(section_beats) == 0: continue start_frame = section_beats[0] end_frame = section_beats[-1] if len( section_beats) > 1 else start_frame + 100 # Calculate average energy for this section section_slice = slice( max(0, start_frame), min(end_frame, len(rms))) section_energy = np.mean( rms[section_slice]) if section_slice.start < section_slice.stop else 0.5 # Generate mood for this section mood = self._generate_mood_description(section_energy, tempo) # Build prompt prompt_parts = [base_prompt] if theme: prompt_parts.append(theme) prompt_parts.append(mood) if style: prompt_parts.append(style) prompts[int(start_frame)] = ", ".join(prompt_parts) # Ensure frame 0 has a prompt if 0 not in prompts: fallback_parts = [base_prompt] if theme: fallback_parts.append(theme) if style: fallback_parts.append(style) prompts[0] = ", ".join(fallback_parts) return prompts def _generate_mood_description(self, energy: float, tempo: float) -> str: """Generate mood description based on audio features""" mood_parts = [] # Energy-based mood if energy < 0.33: mood_parts.append("calm and peaceful") elif energy < 0.66: mood_parts.append("dynamic and flowing") else: mood_parts.append("intense and energetic") # Tempo-based mood if tempo < 90: mood_parts.append("slow and contemplative") elif tempo < 130: mood_parts.append("moderate paced") else: mood_parts.append("fast and driving") return ", ".join(mood_parts) def create_deforum_preset(self, analysis: Dict[str, Any], settings: Dict[str, Any], keyframes: Dict[str, str], prompts: Dict[int, str]) -> Dict[str, Any]: """Create complete Deforum preset JSON""" # Convert prompts to string keys (required by Deforum) animation_prompts = {} for frame_num, prompt_text in sorted(prompts.items()): animation_prompts[str(frame_num)] = prompt_text # Create complete preset structure preset = { # Basic generation settings "W": int(settings["width"]), "H": int(settings["height"]), "seed": int(settings["seed"]), "sampler": settings["sampler"], "steps": int(settings["steps"]), "scale": float(settings["cfg_scale"]), "ddim_eta": 0.0, # Animation settings "animation_mode": "2D", "max_frames": int(analysis["total_frames"]), "border": "replicate", # Camera movement keyframes "zoom": keyframes.get("zoom", "0:(1.0)"), "angle": keyframes.get("angle", "0:(0)"), "translation_x": keyframes.get("translation_x", "0:(0)"), "translation_y": keyframes.get("translation_y", "0:(0)"), "translation_z": "0:(0)", "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0)", # Strength and noise schedules "strength_schedule": keyframes.get("strength_schedule", "0:(0.65)"), "noise_schedule": "0:(0.02)", "amount_schedule": "0:(0.2)", # Color and coherence settings "color_coherence": "Match Frame 0 LAB", "color_force_grayscale": False, # Diffusion settings "diffusion_cadence": "1", # 3D settings (required even for 2D) "use_depth_warping": False, "midas_weight": 0.3, "near_plane": 200, "far_plane": 10000, "fov": 40, "padding_mode": "border", "sampling_mode": "bicubic", # Animation prompts "animation_prompts": animation_prompts, # Output settings "fps": int(analysis["fps"]), "batch_name": "MusicDeforum", "filename_format": "{timestring}_{index}_{prompt}.png", "save_samples": True, "save_settings": True, "display_samples": True, # Video settings "skip_video_for_run_all": False, "make_gif": False, "delete_imgs": False } return preset # Initialize the generator generator = DeforumMusicGenerator() def process_audio(audio_file: Union[str, None], fps: int, width: int, height: int, steps: int, cfg_scale: float, sampler: str, seed: int, base_prompt: str, style: str, theme: str, selected_variants: List[str]) -> Tuple[str, str, Union[str, None]]: """Main processing function with proper type annotations""" if not audio_file: return "❌ Please upload an audio file first.", "", None if not HAS_LIBROSA: return "❌ librosa is not installed. Please run: pip install librosa soundfile", "", None try: # Analyze the audio file status_msg = "🎵 Analyzing audio file...\n" analysis = generator.analyze_audio_file(audio_file, fps) duration = analysis["duration"] tempo = analysis["tempo"] total_frames = analysis["total_frames"] beat_count = len(analysis["beat_frames"]) status_msg += f"✅ Audio Analysis Complete:\n" status_msg += f" Duration: {duration:.2f} seconds\n" status_msg += f" Tempo: {tempo:.1f} BPM\n" status_msg += f" Total Frames: {total_frames}\n" status_msg += f" Beats Detected: {beat_count}\n\n" # Prepare settings settings = { "width": width, "height": height, "fps": fps, "steps": steps, "cfg_scale": cfg_scale, "sampler": sampler, "seed": seed } # Generate presets for selected variants generated_presets = {} temp_files = [] for variant_name in selected_variants: if variant_name not in generator.PRESET_VARIANTS: continue status_msg += f"🎨 Generating {variant_name} preset...\n" variant_settings = generator.PRESET_VARIANTS[variant_name] # Generate keyframes based on audio keyframes = generator.generate_keyframes( analysis, variant_settings) # Generate prompts prompts = generator.generate_prompts( analysis, base_prompt, style, theme, variant_settings["beats_per_section"] ) # Create complete Deforum preset preset = generator.create_deforum_preset( analysis, settings, keyframes, prompts) generated_presets[variant_name] = preset # Save to temporary file temp_file = tempfile.NamedTemporaryFile( mode='w', suffix=f'_{variant_name}.json', delete=False) json.dump(preset, temp_file, indent=2, ensure_ascii=False) temp_file.close() temp_files.append((variant_name, temp_file.name)) if not generated_presets: return "❌ No presets generated. Please select at least one variant.", "", None # Create ZIP file with all presets zip_file_path = tempfile.NamedTemporaryFile( suffix='.zip', delete=False) zip_file_path.close() with zipfile.ZipFile(zip_file_path.name, 'w', zipfile.ZIP_DEFLATED) as zip_file: for variant_name, file_path in temp_files: zip_file.write( file_path, f"deforum_preset_{variant_name}.json") # Create preview of generated presets preview_data = {} for variant_name, preset in generated_presets.items(): preview_data[variant_name] = { "Total Frames": preset["max_frames"], "Dimensions": f"{preset['W']}x{preset['H']}", "Steps": preset["steps"], "CFG Scale": preset["scale"], "Sampler": preset["sampler"], "Prompt Count": len(preset["animation_prompts"]), "First Prompt": list(preset["animation_prompts"].values())[0][:100] + "..." if preset["animation_prompts"] else "None" } status_msg += f"✅ Successfully generated {len(generated_presets)} Deforum presets!\n\n" status_msg += "📥 Download the ZIP file and extract the JSON files.\n" status_msg += "🎬 In Deforum: Load Settings → Select JSON file → Generate!\n" preview_json = json.dumps(preview_data, indent=2) return status_msg, preview_json, zip_file_path.name except Exception as e: error_msg = f"❌ Error processing audio file:\n\n{str(e)}\n\n" error_msg += f"Traceback:\n{traceback.format_exc()}" return error_msg, "", None def create_interface(): """Create Gradio interface""" with gr.Blocks( title="Deforum Music Generator", theme=gr.themes.Soft(), css=""" .main-container { max-width: 1200px; margin: 0 auto; } """ ) as interface: gr.Markdown(""" # 🎵 Deforum Music-to-Video Generator Transform your music into synchronized Deforum animation presets! Upload a music file and generate camera movements that sync to the beat and energy of your track. """) # Dependency status status_msgs = [] if not HAS_LIBROSA: status_msgs.append( "❌ **librosa missing** - Install with: `pip install librosa soundfile`") if not HAS_WHISPER: status_msgs.append( "⚠️ **whisper missing** - Optional for lyrics: `pip install openai-whisper`") if not HAS_OPENAI: status_msgs.append( "⚠️ **openai missing** - Optional for GPT: `pip install openai`") if status_msgs: gr.Markdown("\n".join(status_msgs)) else: gr.Markdown("✅ All dependencies loaded successfully!") with gr.Tab("🎵 Generate Presets"): with gr.Row(): with gr.Column(scale=1): # Audio input audio_input = gr.Audio( label="🎵 Upload Music File", type="filepath" ) # Video settings with gr.Group(): gr.Markdown("### 📹 Video Settings") fps = gr.Slider( minimum=12, maximum=60, value=24, step=1, label="FPS (Frames Per Second)" ) with gr.Row(): width = gr.Slider( minimum=256, maximum=2048, value=1024, step=64, label="Width" ) height = gr.Slider( minimum=256, maximum=1536, value=576, step=64, label="Height" ) # Generation settings with gr.Group(): gr.Markdown("### ⚙️ Generation Settings") steps = gr.Slider( minimum=10, maximum=100, value=30, step=1, label="Steps" ) cfg_scale = gr.Slider( minimum=1.0, maximum=20.0, value=7.0, step=0.5, label="CFG Scale" ) sampler = gr.Dropdown( choices=[ "DPM++ 2M Karras", "Euler a", "DDIM", "LMS Karras", "DPM++ SDE Karras", "PLMS" ], value="DPM++ 2M Karras", label="Sampler" ) seed = gr.Number( value=-1, label="Seed (-1 for random)", precision=0 ) with gr.Column(scale=1): # Prompt settings with gr.Group(): gr.Markdown("### 📝 Prompt Settings") base_prompt = gr.Textbox( value="abstract art, flowing shapes, vibrant colors, cinematic lighting", label="Base Prompt", lines=3, placeholder="Describe the visual style you want...") style = gr.Textbox( value="highly detailed, artistic, professional photography", label="Style Modifiers", lines=2, placeholder="Additional style keywords...") theme = gr.Textbox( value="", label="Theme (Optional)", lines=1, placeholder="e.g., cyberpunk, nature, space, fantasy...") # Variant selection with gr.Group(): gr.Markdown("### 🎭 Preset Variants") variants = gr.CheckboxGroup( choices=["Calm", "Punchy", "Glitch"], value=["Calm", "Punchy"], label="Select Variants to Generate" ) gr.Markdown(""" - **Calm**: Subtle movements, smooth transitions - **Punchy**: Medium intensity, good for most music - **Glitch**: High intensity, chaotic movements """) # Generate button generate_btn = gr.Button( "🚀 Generate Deforum Presets", variant="primary", size="lg" ) # Output section with gr.Row(): with gr.Column(): status_output = gr.Textbox( label="📊 Generation Status", lines=12, interactive=False, show_copy_button=True ) with gr.Column(): preview_output = gr.Code( label="👁️ Preset Preview", language="json", lines=12 ) # Download output download_output = gr.File( label="📥 Download Presets (ZIP file)", interactive=False ) # Connect the generate button generate_btn.click( fn=process_audio, inputs=[ audio_input, fps, width, height, steps, cfg_scale, sampler, seed, base_prompt, style, theme, variants], outputs=[ status_output, preview_output, download_output], show_progress=True) with gr.Tab("📚 Usage Guide"): gr.Markdown(""" ## 🚀 How to Use ### 1. Upload Music - Supported formats: MP3, WAV, FLAC, M4A - Recommended length: 1-5 minutes - The tool analyzes beats, tempo, and energy patterns ### 2. Configure Settings **Video Settings:** - **FPS**: 24 is standard, 30+ for smoother motion - **Dimensions**: 1024x576 is a good balance of quality and speed **Generation Settings:** - **Steps**: 30-40 for good quality, 50+ for best results - **CFG Scale**: 7-10 for good prompt adherence - **Sampler**: DPM++ 2M Karras is reliable and fast **Prompts:** - **Base Prompt**: Your main visual concept - **Style**: Quality and aesthetic modifiers - **Theme**: Optional thematic elements ### 3. Select Variants - **Calm**: Subtle camera movements, dreamy atmosphere - **Punchy**: Dynamic movements that follow the beat - **Glitch**: Intense, chaotic movements for electronic music ### 4. Generate and Use 1. Click "Generate Deforum Presets" 2. Wait for processing (usually 1-2 minutes) 3. Download the ZIP file 4. Extract JSON files 5. In Deforum: Load Settings → Select JSON → Generate! ## 🎨 Tips for Better Results ### Prompt Writing ``` Good: "cyberpunk cityscape, neon lights, rain, cinematic" Better: "cyberpunk cityscape, vibrant neon reflections, volumetric rain, cinematic lighting, highly detailed" ``` ### Music Type Recommendations **Electronic/EDM**: Punchy or Glitch variants ``` Base: "abstract digital art, geometric patterns, vibrant colors" Style: "synthwave aesthetic, neon glow, high contrast" ``` **Rock/Metal**: Punchy or Glitch variants ``` Base: "dramatic landscape, storm clouds, lightning" Style: "dark atmosphere, intense lighting, cinematic" ``` **Ambient/Classical**: Calm variant ``` Base: "ethereal cloudscapes, flowing forms, soft light" Style: "painterly, dreamy, artistic, serene" ``` **Hip-Hop**: Punchy variant ``` Base: "urban environment, street art, dynamic angles" Style: "gritty texture, bold colors, street photography" ``` ## 🔧 Troubleshooting **"librosa not installed"** ```bash pip install librosa soundfile ``` **"Audio analysis failed"** - Check that FFmpeg is installed - Try a different audio format - Ensure file isn't corrupted **"Preset won't load in Deforum"** - Make sure you have the latest Deforum extension - Try loading the JSON in a text editor to check for errors - Verify all required fields are present **Poor synchronization** - Try different FPS settings - Ensure your music has clear beats - Consider the "Punchy" variant for better beat detection """) return interface # Main execution if __name__ == "__main__": print("🎵 Starting Deforum Music Generator...") print(f"Gradio version: {gr.__version__}") # Create and launch the interface app = create_interface() # Launch with corrected settings try: app.launch( share=True, # Set to False for local testing server_name="0.0.0.0", # Allow external connections server_port=7861, show_error=True, quiet=False, show_api=False # Disable API docs to avoid schema issues ) except Exception as e: print(f"❌ Launch failed: {e}") print("Trying fallback launch configuration...") app.launch( share=False, inbrowser=True, show_error=True )