Spaces:

gulle1155
/

Scripts

Sleeping

Scripts / deforum_gradio3_compatible.py

Upload folder using huggingface_hub

18c9c13 verified about 1 year ago

30.5 kB

#!/usr/bin/env python3
"""
Deforum Music-to-Video Interface - Fixed for Gradio 4.44.1
"""

import math
import collections
import re
import gradio as gr
import os
import sys
import json
import traceback
import subprocess
import importlib.util
from pathlib import Path
import tempfile
import zipfile
from typing import Optional, Dict, Any, List, Tuple, Union
import warnings
warnings.filterwarnings("ignore")

# Import dependencies with proper error handling
print("Loading dependencies...")

try:
    import numpy as np
    print("✅ numpy loaded")
except ImportError:
    print("❌ numpy not found")
    sys.exit(1)

try:
    import librosa
    HAS_LIBROSA = True
    print("✅ librosa loaded")
except ImportError:
    HAS_LIBROSA = False
    print("❌ librosa not found - please install: pip install librosa soundfile")

try:
    import whisper
    HAS_WHISPER = True
    print("✅ whisper loaded")
except ImportError:
    HAS_WHISPER = False
    print("⚠️ whisper not found - transcription disabled")

try:
    import openai
    HAS_OPENAI = True
    print("✅ openai loaded")
except ImportError:
    HAS_OPENAI = False
    print("⚠️ openai not found - GPT expansion disabled")

try:
    import requests
    print("✅ requests loaded")
except ImportError:
    print("❌ requests not found")


class DeforumMusicGenerator:
    def __init__(self):
        self.PRESET_VARIANTS = {
            "Calm": {
                "zoom_amt": 0.012,
                "rot_amt": 0.35,
                "trans_amt": 0.06,
                "denoise_base": 0.30,
                "beats_per_section": 32
            },
            "Punchy": {
                "zoom_amt": 0.024,
                "rot_amt": 0.9,
                "trans_amt": 0.15,
                "denoise_base": 0.36,
                "beats_per_section": 16
            },
            "Glitch": {
                "zoom_amt": 0.03,
                "rot_amt": 1.2,
                "trans_amt": 0.22,
                "denoise_base": 0.42,
                "beats_per_section": 8
            }
        }

    def analyze_audio_file(self, audio_path: str,
                           fps: int = 24) -> Dict[str, Any]:
        """Analyze audio file and extract musical features"""
        if not HAS_LIBROSA:
            raise RuntimeError(
                "librosa is required for audio analysis. Install with: pip install librosa soundfile")

        try:
            # Load audio file
            y, sr = librosa.load(audio_path, sr=None, mono=True)
            duration = float(librosa.get_duration(y=y, sr=sr))

            # Extract audio features
            hop_length = 512

            # RMS energy (volume/intensity)
            rms = librosa.feature.rms(
                y=y, frame_length=2048, hop_length=hop_length).flatten()

            # Spectral centroid (brightness)
            spectral_centroid = librosa.feature.spectral_centroid(
                y=y, sr=sr, hop_length=hop_length).flatten()

            # Beat tracking
            tempo, beat_frames = librosa.beat.beat_track(
                y=y, sr=sr, hop_length=hop_length)
            beat_times = librosa.frames_to_time(
                beat_frames, sr=sr, hop_length=hop_length)

            # Convert to video frame numbers
            total_frames = int(duration * fps)
            beat_frame_numbers = np.round(beat_times * fps).astype(int)

            # Interpolate audio features to video timeline
            audio_times = librosa.frames_to_time(
                np.arange(len(rms)), sr=sr, hop_length=hop_length)
            video_times = np.arange(total_frames) / fps

            # Interpolate and normalize features
            rms_video = np.interp(video_times, audio_times, rms)
            brightness_video = np.interp(
                video_times, audio_times, spectral_centroid)

            # Normalize to 0-1 range
            rms_normalized = self._normalize_array(rms_video)
            brightness_normalized = self._normalize_array(brightness_video)

            # Apply smoothing
            rms_smooth = self._smooth_array(rms_normalized)
            brightness_smooth = self._smooth_array(brightness_normalized)

            return {
                "duration": duration,
                "fps": fps,
                "total_frames": total_frames,
                "tempo": float(tempo),
                "beat_frames": beat_frame_numbers,
                "rms_smooth": rms_smooth,
                "brightness_smooth": brightness_smooth,
                "sr": sr
            }

        except Exception as e:
            raise RuntimeError(f"Audio analysis failed: {str(e)}")

    def _normalize_array(self, arr: np.ndarray) -> np.ndarray:
        """Normalize array to 0-1 range"""
        arr = np.asarray(arr, dtype=float)
        min_val, max_val = arr.min(), arr.max()
        if max_val - min_val < 1e-8:
            return np.zeros_like(arr)
        return (arr - min_val) / (max_val - min_val)

    def _smooth_array(self, arr: np.ndarray, window: int = 8) -> np.ndarray:
        """Apply moving average smoothing"""
        if window <= 1:
            return arr
        kernel = np.ones(window) / window
        return np.convolve(arr, kernel, mode='same')

    def generate_keyframes(self,
                           analysis: Dict[str,
                                          Any],
                           variant_settings: Dict[str,
                                                  float]) -> Dict[str,
                                                                  str]:
        """Generate camera movement keyframes based on audio analysis"""

        beat_frames = analysis["beat_frames"]
        rms = analysis["rms_smooth"]
        total_frames = analysis["total_frames"]

        # Initialize keyframe lists
        zoom_keyframes = []
        angle_keyframes = []
        tx_keyframes = []
        ty_keyframes = []
        strength_keyframes = []

        # Generate keyframes at beat positions
        for frame in beat_frames:
            if frame >= total_frames:
                continue

            # Get audio energy at this frame
            energy = rms[min(frame, len(rms) - 1)]

            # Calculate movement values based on energy
            zoom_factor = 1.0 + (energy - 0.5) * 2.0 * \
                variant_settings["zoom_amt"]
            rotation = (energy - 0.5) * 2.0 * variant_settings["rot_amt"]
            translation = (energy - 0.5) * 2.0 * variant_settings["trans_amt"]

            # Strength (denoising) - higher energy = lower strength for more
            # variation
            strength = max(
                0.1, min(0.9, variant_settings["denoise_base"] + (0.5 - energy) * 0.25))

            # Add keyframes
            zoom_keyframes.append((frame, zoom_factor))
            angle_keyframes.append((frame, rotation))
            tx_keyframes.append((frame, translation))
            # Slightly different for Y
            ty_keyframes.append((frame, translation * 0.7))
            strength_keyframes.append((frame, strength))

        # Convert to Deforum keyframe format
        return {
            "zoom": self._keyframes_to_string(zoom_keyframes),
            "angle": self._keyframes_to_string(angle_keyframes),
            "translation_x": self._keyframes_to_string(tx_keyframes),
            "translation_y": self._keyframes_to_string(ty_keyframes),
            "strength_schedule": self._keyframes_to_string(
                strength_keyframes,
                precision=3)}

    def _keyframes_to_string(
            self, keyframes: List[Tuple[int, float]], precision: int = 4) -> str:
        """Convert keyframe list to Deforum string format"""
        if not keyframes:
            return "0:(1.0)" if precision == 4 else "0:(0.65)"

        sorted_kf = sorted(keyframes, key=lambda x: x[0])
        return ", ".join(
            f"{frame}:({value:.{precision}f})" for frame,
            value in sorted_kf)

    def generate_prompts(self,
                         analysis: Dict[str,
                                        Any],
                         base_prompt: str,
                         style: str,
                         theme: str,
                         beats_per_section: int = 16) -> Dict[int,
                                                              str]:
        """Generate animation prompts based on audio sections"""

        beat_frames = analysis["beat_frames"]
        rms = analysis["rms_smooth"]
        tempo = analysis["tempo"]
        total_frames = analysis["total_frames"]

        prompts = {}

        if len(beat_frames) < beats_per_section:
            # Short audio - single prompt
            energy = np.mean(rms)
            mood = self._generate_mood_description(energy, tempo)
            prompt_parts = [base_prompt]
            if theme:
                prompt_parts.append(theme)
            prompt_parts.append(mood)
            if style:
                prompt_parts.append(style)
            prompts[0] = ", ".join(prompt_parts)
        else:
            # Longer audio - section-based prompts
            for i in range(0, len(beat_frames), beats_per_section):
                section_beats = beat_frames[i:i + beats_per_section]
                if section_beats is None or len(section_beats) == 0:
                    continue

                start_frame = section_beats[0]
                end_frame = section_beats[-1] if len(
                    section_beats) > 1 else start_frame + 100

                # Calculate average energy for this section
                section_slice = slice(
                    max(0, start_frame), min(end_frame, len(rms)))
                section_energy = np.mean(
                    rms[section_slice]) if section_slice.start < section_slice.stop else 0.5

                # Generate mood for this section
                mood = self._generate_mood_description(section_energy, tempo)

                # Build prompt
                prompt_parts = [base_prompt]
                if theme:
                    prompt_parts.append(theme)
                prompt_parts.append(mood)
                if style:
                    prompt_parts.append(style)

                prompts[int(start_frame)] = ", ".join(prompt_parts)

        # Ensure frame 0 has a prompt
        if 0 not in prompts:
            fallback_parts = [base_prompt]
            if theme:
                fallback_parts.append(theme)
            if style:
                fallback_parts.append(style)
            prompts[0] = ", ".join(fallback_parts)

        return prompts

    def _generate_mood_description(self, energy: float, tempo: float) -> str:
        """Generate mood description based on audio features"""
        mood_parts = []

        # Energy-based mood
        if energy < 0.33:
            mood_parts.append("calm and peaceful")
        elif energy < 0.66:
            mood_parts.append("dynamic and flowing")
        else:
            mood_parts.append("intense and energetic")

        # Tempo-based mood
        if tempo < 90:
            mood_parts.append("slow and contemplative")
        elif tempo < 130:
            mood_parts.append("moderate paced")
        else:
            mood_parts.append("fast and driving")

        return ", ".join(mood_parts)

    def create_deforum_preset(self,
                              analysis: Dict[str,
                                             Any],
                              settings: Dict[str,
                                             Any],
                              keyframes: Dict[str,
                                              str],
                              prompts: Dict[int,
                                            str]) -> Dict[str,
                                                          Any]:
        """Create complete Deforum preset JSON"""

        # Convert prompts to string keys (required by Deforum)
        animation_prompts = {}
        for frame_num, prompt_text in sorted(prompts.items()):
            animation_prompts[str(frame_num)] = prompt_text

        # Create complete preset structure
        preset = {
            # Basic generation settings
            "W": int(settings["width"]),
            "H": int(settings["height"]),
            "seed": int(settings["seed"]),
            "sampler": settings["sampler"],
            "steps": int(settings["steps"]),
            "scale": float(settings["cfg_scale"]),
            "ddim_eta": 0.0,

            # Animation settings
            "animation_mode": "2D",
            "max_frames": int(analysis["total_frames"]),
            "border": "replicate",

            # Camera movement keyframes
            "zoom": keyframes.get("zoom", "0:(1.0)"),
            "angle": keyframes.get("angle", "0:(0)"),
            "translation_x": keyframes.get("translation_x", "0:(0)"),
            "translation_y": keyframes.get("translation_y", "0:(0)"),
            "translation_z": "0:(0)",
            "rotation_3d_x": "0:(0)",
            "rotation_3d_y": "0:(0)",
            "rotation_3d_z": "0:(0)",

            # Strength and noise schedules
            "strength_schedule": keyframes.get("strength_schedule", "0:(0.65)"),
            "noise_schedule": "0:(0.02)",
            "amount_schedule": "0:(0.2)",

            # Color and coherence settings
            "color_coherence": "Match Frame 0 LAB",
            "color_force_grayscale": False,

            # Diffusion settings
            "diffusion_cadence": "1",

            # 3D settings (required even for 2D)
            "use_depth_warping": False,
            "midas_weight": 0.3,
            "near_plane": 200,
            "far_plane": 10000,
            "fov": 40,
            "padding_mode": "border",
            "sampling_mode": "bicubic",

            # Animation prompts
            "animation_prompts": animation_prompts,

            # Output settings
            "fps": int(analysis["fps"]),
            "batch_name": "MusicDeforum",
            "filename_format": "{timestring}_{index}_{prompt}.png",
            "save_samples": True,
            "save_settings": True,
            "display_samples": True,

            # Video settings
            "skip_video_for_run_all": False,
            "make_gif": False,
            "delete_imgs": False
        }

        return preset


# Initialize the generator
generator = DeforumMusicGenerator()


def process_audio(audio_file: Union[str,
                                    None],
                  fps: int,
                  width: int,
                  height: int,
                  steps: int,
                  cfg_scale: float,
                  sampler: str,
                  seed: int,
                  base_prompt: str,
                  style: str,
                  theme: str,
                  selected_variants: List[str]) -> Tuple[str,
                                                         str,
                                                         Union[str,
                                                               None]]:
    """Main processing function with proper type annotations"""

    if not audio_file:
        return "❌ Please upload an audio file first.", "", None

    if not HAS_LIBROSA:
        return "❌ librosa is not installed. Please run: pip install librosa soundfile", "", None

    try:
        # Analyze the audio file
        status_msg = "🎵 Analyzing audio file...\n"
        analysis = generator.analyze_audio_file(audio_file, fps)

        duration = analysis["duration"]
        tempo = analysis["tempo"]
        total_frames = analysis["total_frames"]
        beat_count = len(analysis["beat_frames"])

        status_msg += f"✅ Audio Analysis Complete:\n"
        status_msg += f"   Duration: {duration:.2f} seconds\n"
        status_msg += f"   Tempo: {tempo:.1f} BPM\n"
        status_msg += f"   Total Frames: {total_frames}\n"
        status_msg += f"   Beats Detected: {beat_count}\n\n"

        # Prepare settings
        settings = {
            "width": width,
            "height": height,
            "fps": fps,
            "steps": steps,
            "cfg_scale": cfg_scale,
            "sampler": sampler,
            "seed": seed
        }

        # Generate presets for selected variants
        generated_presets = {}
        temp_files = []

        for variant_name in selected_variants:
            if variant_name not in generator.PRESET_VARIANTS:
                continue

            status_msg += f"🎨 Generating {variant_name} preset...\n"

            variant_settings = generator.PRESET_VARIANTS[variant_name]

            # Generate keyframes based on audio
            keyframes = generator.generate_keyframes(
                analysis, variant_settings)

            # Generate prompts
            prompts = generator.generate_prompts(
                analysis, base_prompt, style, theme,
                variant_settings["beats_per_section"]
            )

            # Create complete Deforum preset
            preset = generator.create_deforum_preset(
                analysis, settings, keyframes, prompts)
            generated_presets[variant_name] = preset

            # Save to temporary file
            temp_file = tempfile.NamedTemporaryFile(
                mode='w', suffix=f'_{variant_name}.json', delete=False)
            json.dump(preset, temp_file, indent=2, ensure_ascii=False)
            temp_file.close()
            temp_files.append((variant_name, temp_file.name))

        if not generated_presets:
            return "❌ No presets generated. Please select at least one variant.", "", None

        # Create ZIP file with all presets
        zip_file_path = tempfile.NamedTemporaryFile(
            suffix='.zip', delete=False)
        zip_file_path.close()

        with zipfile.ZipFile(zip_file_path.name, 'w', zipfile.ZIP_DEFLATED) as zip_file:
            for variant_name, file_path in temp_files:
                zip_file.write(
                    file_path, f"deforum_preset_{variant_name}.json")

        # Create preview of generated presets
        preview_data = {}
        for variant_name, preset in generated_presets.items():
            preview_data[variant_name] = {
                "Total Frames": preset["max_frames"],
                "Dimensions": f"{preset['W']}x{preset['H']}",
                "Steps": preset["steps"],
                "CFG Scale": preset["scale"],
                "Sampler": preset["sampler"],
                "Prompt Count": len(preset["animation_prompts"]),
                "First Prompt": list(preset["animation_prompts"].values())[0][:100] + "..." if preset["animation_prompts"] else "None"
            }

        status_msg += f"✅ Successfully generated {len(generated_presets)} Deforum presets!\n\n"
        status_msg += "📥 Download the ZIP file and extract the JSON files.\n"
        status_msg += "🎬 In Deforum: Load Settings → Select JSON file → Generate!\n"

        preview_json = json.dumps(preview_data, indent=2)

        return status_msg, preview_json, zip_file_path.name

    except Exception as e:
        error_msg = f"❌ Error processing audio file:\n\n{str(e)}\n\n"
        error_msg += f"Traceback:\n{traceback.format_exc()}"
        return error_msg, "", None


def create_interface():
    """Create Gradio interface"""

    with gr.Blocks(
        title="Deforum Music Generator",
        theme=gr.themes.Soft(),
        css="""
        .main-container {
            max-width: 1200px;
            margin: 0 auto;
        }
        """
    ) as interface:

        gr.Markdown("""
        # 🎵 Deforum Music-to-Video Generator
        Transform your music into synchronized Deforum animation presets!
        Upload a music file and generate camera movements that sync to the beat and energy of your track.
        """)

        # Dependency status
        status_msgs = []
        if not HAS_LIBROSA:
            status_msgs.append(
                "❌ **librosa missing** - Install with: `pip install librosa soundfile`")
        if not HAS_WHISPER:
            status_msgs.append(
                "⚠️ **whisper missing** - Optional for lyrics: `pip install openai-whisper`")
        if not HAS_OPENAI:
            status_msgs.append(
                "⚠️ **openai missing** - Optional for GPT: `pip install openai`")

        if status_msgs:
            gr.Markdown("\n".join(status_msgs))
        else:
            gr.Markdown("✅ All dependencies loaded successfully!")

        with gr.Tab("🎵 Generate Presets"):

            with gr.Row():
                with gr.Column(scale=1):

                    # Audio input
                    audio_input = gr.Audio(
                        label="🎵 Upload Music File",
                        type="filepath"
                    )

                    # Video settings
                    with gr.Group():
                        gr.Markdown("### 📹 Video Settings")

                        fps = gr.Slider(
                            minimum=12, maximum=60, value=24, step=1,
                            label="FPS (Frames Per Second)"
                        )

                        with gr.Row():
                            width = gr.Slider(
                                minimum=256, maximum=2048, value=1024, step=64,
                                label="Width"
                            )
                            height = gr.Slider(
                                minimum=256, maximum=1536, value=576, step=64,
                                label="Height"
                            )

                    # Generation settings
                    with gr.Group():
                        gr.Markdown("### ⚙️ Generation Settings")

                        steps = gr.Slider(
                            minimum=10, maximum=100, value=30, step=1,
                            label="Steps"
                        )

                        cfg_scale = gr.Slider(
                            minimum=1.0, maximum=20.0, value=7.0, step=0.5,
                            label="CFG Scale"
                        )

                        sampler = gr.Dropdown(
                            choices=[
                                "DPM++ 2M Karras",
                                "Euler a",
                                "DDIM",
                                "LMS Karras",
                                "DPM++ SDE Karras",
                                "PLMS"
                            ],
                            value="DPM++ 2M Karras",
                            label="Sampler"
                        )

                        seed = gr.Number(
                            value=-1,
                            label="Seed (-1 for random)",
                            precision=0
                        )

                with gr.Column(scale=1):

                    # Prompt settings
                    with gr.Group():
                        gr.Markdown("### 📝 Prompt Settings")

                        base_prompt = gr.Textbox(
                            value="abstract art, flowing shapes, vibrant colors, cinematic lighting",
                            label="Base Prompt",
                            lines=3,
                            placeholder="Describe the visual style you want...")

                        style = gr.Textbox(
                            value="highly detailed, artistic, professional photography",
                            label="Style Modifiers",
                            lines=2,
                            placeholder="Additional style keywords...")

                        theme = gr.Textbox(
                            value="",
                            label="Theme (Optional)",
                            lines=1,
                            placeholder="e.g., cyberpunk, nature, space, fantasy...")

                    # Variant selection
                    with gr.Group():
                        gr.Markdown("### 🎭 Preset Variants")

                        variants = gr.CheckboxGroup(
                            choices=["Calm", "Punchy", "Glitch"],
                            value=["Calm", "Punchy"],
                            label="Select Variants to Generate"
                        )

                        gr.Markdown("""
                        - **Calm**: Subtle movements, smooth transitions
                        - **Punchy**: Medium intensity, good for most music
                        - **Glitch**: High intensity, chaotic movements
                        """)

            # Generate button
            generate_btn = gr.Button(
                "🚀 Generate Deforum Presets",
                variant="primary",
                size="lg"
            )

            # Output section
            with gr.Row():
                with gr.Column():
                    status_output = gr.Textbox(
                        label="📊 Generation Status",
                        lines=12,
                        interactive=False,
                        show_copy_button=True
                    )

                with gr.Column():
                    preview_output = gr.Code(
                        label="👁️ Preset Preview",
                        language="json",
                        lines=12
                    )

            # Download output
            download_output = gr.File(
                label="📥 Download Presets (ZIP file)",
                interactive=False
            )

            # Connect the generate button
            generate_btn.click(
                fn=process_audio,
                inputs=[
                    audio_input,
                    fps,
                    width,
                    height,
                    steps,
                    cfg_scale,
                    sampler,
                    seed,
                    base_prompt,
                    style,
                    theme,
                    variants],
                outputs=[
                    status_output,
                    preview_output,
                    download_output],
                show_progress=True)

        with gr.Tab("📚 Usage Guide"):
            gr.Markdown("""
            ## 🚀 How to Use
            ### 1. Upload Music
            - Supported formats: MP3, WAV, FLAC, M4A
            - Recommended length: 1-5 minutes
            - The tool analyzes beats, tempo, and energy patterns
            ### 2. Configure Settings
            **Video Settings:**
            - **FPS**: 24 is standard, 30+ for smoother motion
            - **Dimensions**: 1024x576 is a good balance of quality and speed
            **Generation Settings:**
            - **Steps**: 30-40 for good quality, 50+ for best results
            - **CFG Scale**: 7-10 for good prompt adherence
            - **Sampler**: DPM++ 2M Karras is reliable and fast
            **Prompts:**
            - **Base Prompt**: Your main visual concept
            - **Style**: Quality and aesthetic modifiers
            - **Theme**: Optional thematic elements
            ### 3. Select Variants
            - **Calm**: Subtle camera movements, dreamy atmosphere
            - **Punchy**: Dynamic movements that follow the beat
            - **Glitch**: Intense, chaotic movements for electronic music
            ### 4. Generate and Use
            1. Click "Generate Deforum Presets"
            2. Wait for processing (usually 1-2 minutes)
            3. Download the ZIP file
            4. Extract JSON files
            5. In Deforum: Load Settings → Select JSON → Generate!
            ## 🎨 Tips for Better Results
            ### Prompt Writing
            ```
            Good: "cyberpunk cityscape, neon lights, rain, cinematic"
            Better: "cyberpunk cityscape, vibrant neon reflections, volumetric rain, cinematic lighting, highly detailed"
            ```
            ### Music Type Recommendations
            **Electronic/EDM**: Punchy or Glitch variants
            ```
            Base: "abstract digital art, geometric patterns, vibrant colors"
            Style: "synthwave aesthetic, neon glow, high contrast"
            ```
            **Rock/Metal**: Punchy or Glitch variants
            ```
            Base: "dramatic landscape, storm clouds, lightning"
            Style: "dark atmosphere, intense lighting, cinematic"
            ```
            **Ambient/Classical**: Calm variant
            ```
            Base: "ethereal cloudscapes, flowing forms, soft light"
            Style: "painterly, dreamy, artistic, serene"
            ```
            **Hip-Hop**: Punchy variant
            ```
            Base: "urban environment, street art, dynamic angles"
            Style: "gritty texture, bold colors, street photography"
            ```
            ## 🔧 Troubleshooting
            **"librosa not installed"**
            ```bash
            pip install librosa soundfile
            ```
            **"Audio analysis failed"**
            - Check that FFmpeg is installed
            - Try a different audio format
            - Ensure file isn't corrupted
            **"Preset won't load in Deforum"**
            - Make sure you have the latest Deforum extension
            - Try loading the JSON in a text editor to check for errors
            - Verify all required fields are present
            **Poor synchronization**
            - Try different FPS settings
            - Ensure your music has clear beats
            - Consider the "Punchy" variant for better beat detection
            """)

    return interface


# Main execution
if __name__ == "__main__":
    print("🎵 Starting Deforum Music Generator...")
    print(f"Gradio version: {gr.__version__}")

    # Create and launch the interface
    app = create_interface()

    # Launch with corrected settings
    try:
        app.launch(
            share=True,  # Set to False for local testing
            server_name="0.0.0.0",  # Allow external connections
            server_port=7861,
            show_error=True,
            quiet=False,
            show_api=False  # Disable API docs to avoid schema issues
        )
    except Exception as e:
        print(f"❌ Launch failed: {e}")
        print("Trying fallback launch configuration...")
        app.launch(
            share=False,
            inbrowser=True,
            show_error=True
        )

	#!/usr/bin/env python3
	"""
	Deforum Music-to-Video Interface - Fixed for Gradio 4.44.1
	"""

	import math
	import collections
	import re
	import gradio as gr
	import os
	import sys
	import json
	import traceback
	import subprocess
	import importlib.util
	from pathlib import Path
	import tempfile
	import zipfile
	from typing import Optional, Dict, Any, List, Tuple, Union
	import warnings
	warnings.filterwarnings("ignore")

	# Import dependencies with proper error handling
	print("Loading dependencies...")

	try:
	import numpy as np
	print("✅ numpy loaded")
	except ImportError:
	print("❌ numpy not found")
	sys.exit(1)

	try:
	import librosa
	HAS_LIBROSA = True
	print("✅ librosa loaded")
	except ImportError:
	HAS_LIBROSA = False
	print("❌ librosa not found - please install: pip install librosa soundfile")

	try:
	import whisper
	HAS_WHISPER = True
	print("✅ whisper loaded")
	except ImportError:
	HAS_WHISPER = False
	print("⚠️ whisper not found - transcription disabled")

	try:
	import openai
	HAS_OPENAI = True
	print("✅ openai loaded")
	except ImportError:
	HAS_OPENAI = False
	print("⚠️ openai not found - GPT expansion disabled")

	try:
	import requests
	print("✅ requests loaded")
	except ImportError:
	print("❌ requests not found")


	class DeforumMusicGenerator:
	def __init__(self):
	self.PRESET_VARIANTS = {
	"Calm": {
	"zoom_amt": 0.012,
	"rot_amt": 0.35,
	"trans_amt": 0.06,
	"denoise_base": 0.30,
	"beats_per_section": 32
	},
	"Punchy": {
	"zoom_amt": 0.024,
	"rot_amt": 0.9,
	"trans_amt": 0.15,
	"denoise_base": 0.36,
	"beats_per_section": 16
	},
	"Glitch": {
	"zoom_amt": 0.03,
	"rot_amt": 1.2,
	"trans_amt": 0.22,
	"denoise_base": 0.42,
	"beats_per_section": 8
	}
	}

	def analyze_audio_file(self, audio_path: str,
	fps: int = 24) -> Dict[str, Any]:
	"""Analyze audio file and extract musical features"""
	if not HAS_LIBROSA:
	raise RuntimeError(
	"librosa is required for audio analysis. Install with: pip install librosa soundfile")

	try:
	# Load audio file
	y, sr = librosa.load(audio_path, sr=None, mono=True)
	duration = float(librosa.get_duration(y=y, sr=sr))

	# Extract audio features
	hop_length = 512

	# RMS energy (volume/intensity)
	rms = librosa.feature.rms(
	y=y, frame_length=2048, hop_length=hop_length).flatten()

	# Spectral centroid (brightness)
	spectral_centroid = librosa.feature.spectral_centroid(
	y=y, sr=sr, hop_length=hop_length).flatten()

	# Beat tracking
	tempo, beat_frames = librosa.beat.beat_track(
	y=y, sr=sr, hop_length=hop_length)
	beat_times = librosa.frames_to_time(
	beat_frames, sr=sr, hop_length=hop_length)

	# Convert to video frame numbers
	total_frames = int(duration * fps)
	beat_frame_numbers = np.round(beat_times * fps).astype(int)

	# Interpolate audio features to video timeline
	audio_times = librosa.frames_to_time(
	np.arange(len(rms)), sr=sr, hop_length=hop_length)
	video_times = np.arange(total_frames) / fps

	# Interpolate and normalize features
	rms_video = np.interp(video_times, audio_times, rms)
	brightness_video = np.interp(
	video_times, audio_times, spectral_centroid)

	# Normalize to 0-1 range
	rms_normalized = self._normalize_array(rms_video)
	brightness_normalized = self._normalize_array(brightness_video)

	# Apply smoothing
	rms_smooth = self._smooth_array(rms_normalized)
	brightness_smooth = self._smooth_array(brightness_normalized)

	return {
	"duration": duration,
	"fps": fps,
	"total_frames": total_frames,
	"tempo": float(tempo),
	"beat_frames": beat_frame_numbers,
	"rms_smooth": rms_smooth,
	"brightness_smooth": brightness_smooth,
	"sr": sr
	}

	except Exception as e:
	raise RuntimeError(f"Audio analysis failed: {str(e)}")

	def _normalize_array(self, arr: np.ndarray) -> np.ndarray:
	"""Normalize array to 0-1 range"""
	arr = np.asarray(arr, dtype=float)
	min_val, max_val = arr.min(), arr.max()
	if max_val - min_val < 1e-8:
	return np.zeros_like(arr)
	return (arr - min_val) / (max_val - min_val)

	def _smooth_array(self, arr: np.ndarray, window: int = 8) -> np.ndarray:
	"""Apply moving average smoothing"""
	if window <= 1:
	return arr
	kernel = np.ones(window) / window
	return np.convolve(arr, kernel, mode='same')

	def generate_keyframes(self,
	analysis: Dict[str,
	Any],
	variant_settings: Dict[str,
	float]) -> Dict[str,
	str]:
	"""Generate camera movement keyframes based on audio analysis"""

	beat_frames = analysis["beat_frames"]
	rms = analysis["rms_smooth"]
	total_frames = analysis["total_frames"]

	# Initialize keyframe lists
	zoom_keyframes = []
	angle_keyframes = []
	tx_keyframes = []
	ty_keyframes = []
	strength_keyframes = []

	# Generate keyframes at beat positions
	for frame in beat_frames:
	if frame >= total_frames:
	continue

	# Get audio energy at this frame
	energy = rms[min(frame, len(rms) - 1)]

	# Calculate movement values based on energy
	zoom_factor = 1.0 + (energy - 0.5) * 2.0 * \
	variant_settings["zoom_amt"]
	rotation = (energy - 0.5) * 2.0 * variant_settings["rot_amt"]
	translation = (energy - 0.5) * 2.0 * variant_settings["trans_amt"]

	# Strength (denoising) - higher energy = lower strength for more
	# variation
	strength = max(
	0.1, min(0.9, variant_settings["denoise_base"] + (0.5 - energy) * 0.25))

	# Add keyframes
	zoom_keyframes.append((frame, zoom_factor))
	angle_keyframes.append((frame, rotation))
	tx_keyframes.append((frame, translation))
	# Slightly different for Y
	ty_keyframes.append((frame, translation * 0.7))
	strength_keyframes.append((frame, strength))

	# Convert to Deforum keyframe format
	return {
	"zoom": self._keyframes_to_string(zoom_keyframes),
	"angle": self._keyframes_to_string(angle_keyframes),
	"translation_x": self._keyframes_to_string(tx_keyframes),
	"translation_y": self._keyframes_to_string(ty_keyframes),
	"strength_schedule": self._keyframes_to_string(
	strength_keyframes,
	precision=3)}

	def _keyframes_to_string(
	self, keyframes: List[Tuple[int, float]], precision: int = 4) -> str:
	"""Convert keyframe list to Deforum string format"""
	if not keyframes:
	return "0:(1.0)" if precision == 4 else "0:(0.65)"

	sorted_kf = sorted(keyframes, key=lambda x: x[0])
	return ", ".join(
	f"{frame}:({value:.{precision}f})" for frame,
	value in sorted_kf)

	def generate_prompts(self,
	analysis: Dict[str,
	Any],
	base_prompt: str,
	style: str,
	theme: str,
	beats_per_section: int = 16) -> Dict[int,
	str]:
	"""Generate animation prompts based on audio sections"""

	beat_frames = analysis["beat_frames"]
	rms = analysis["rms_smooth"]
	tempo = analysis["tempo"]
	total_frames = analysis["total_frames"]

	prompts = {}

	if len(beat_frames) < beats_per_section:
	# Short audio - single prompt
	energy = np.mean(rms)
	mood = self._generate_mood_description(energy, tempo)
	prompt_parts = [base_prompt]
	if theme:
	prompt_parts.append(theme)
	prompt_parts.append(mood)
	if style:
	prompt_parts.append(style)
	prompts[0] = ", ".join(prompt_parts)
	else:
	# Longer audio - section-based prompts
	for i in range(0, len(beat_frames), beats_per_section):
	section_beats = beat_frames[i:i + beats_per_section]
	if section_beats is None or len(section_beats) == 0:
	continue

	start_frame = section_beats[0]
	end_frame = section_beats[-1] if len(
	section_beats) > 1 else start_frame + 100

	# Calculate average energy for this section
	section_slice = slice(
	max(0, start_frame), min(end_frame, len(rms)))
	section_energy = np.mean(
	rms[section_slice]) if section_slice.start < section_slice.stop else 0.5

	# Generate mood for this section
	mood = self._generate_mood_description(section_energy, tempo)

	# Build prompt
	prompt_parts = [base_prompt]
	if theme:
	prompt_parts.append(theme)
	prompt_parts.append(mood)
	if style:
	prompt_parts.append(style)

	prompts[int(start_frame)] = ", ".join(prompt_parts)

	# Ensure frame 0 has a prompt
	if 0 not in prompts:
	fallback_parts = [base_prompt]
	if theme:
	fallback_parts.append(theme)
	if style:
	fallback_parts.append(style)
	prompts[0] = ", ".join(fallback_parts)

	return prompts

	def _generate_mood_description(self, energy: float, tempo: float) -> str:
	"""Generate mood description based on audio features"""
	mood_parts = []

	# Energy-based mood
	if energy < 0.33:
	mood_parts.append("calm and peaceful")
	elif energy < 0.66:
	mood_parts.append("dynamic and flowing")
	else:
	mood_parts.append("intense and energetic")

	# Tempo-based mood
	if tempo < 90:
	mood_parts.append("slow and contemplative")
	elif tempo < 130:
	mood_parts.append("moderate paced")
	else:
	mood_parts.append("fast and driving")

	return ", ".join(mood_parts)

	def create_deforum_preset(self,
	analysis: Dict[str,
	Any],
	settings: Dict[str,
	Any],
	keyframes: Dict[str,
	str],
	prompts: Dict[int,
	str]) -> Dict[str,
	Any]:
	"""Create complete Deforum preset JSON"""

	# Convert prompts to string keys (required by Deforum)
	animation_prompts = {}
	for frame_num, prompt_text in sorted(prompts.items()):
	animation_prompts[str(frame_num)] = prompt_text

	# Create complete preset structure
	preset = {
	# Basic generation settings
	"W": int(settings["width"]),
	"H": int(settings["height"]),
	"seed": int(settings["seed"]),
	"sampler": settings["sampler"],
	"steps": int(settings["steps"]),
	"scale": float(settings["cfg_scale"]),
	"ddim_eta": 0.0,

	# Animation settings
	"animation_mode": "2D",
	"max_frames": int(analysis["total_frames"]),
	"border": "replicate",

	# Camera movement keyframes
	"zoom": keyframes.get("zoom", "0:(1.0)"),
	"angle": keyframes.get("angle", "0:(0)"),
	"translation_x": keyframes.get("translation_x", "0:(0)"),
	"translation_y": keyframes.get("translation_y", "0:(0)"),
	"translation_z": "0:(0)",
	"rotation_3d_x": "0:(0)",
	"rotation_3d_y": "0:(0)",
	"rotation_3d_z": "0:(0)",

	# Strength and noise schedules
	"strength_schedule": keyframes.get("strength_schedule", "0:(0.65)"),
	"noise_schedule": "0:(0.02)",
	"amount_schedule": "0:(0.2)",

	# Color and coherence settings
	"color_coherence": "Match Frame 0 LAB",
	"color_force_grayscale": False,

	# Diffusion settings
	"diffusion_cadence": "1",

	# 3D settings (required even for 2D)
	"use_depth_warping": False,
	"midas_weight": 0.3,
	"near_plane": 200,
	"far_plane": 10000,
	"fov": 40,
	"padding_mode": "border",
	"sampling_mode": "bicubic",

	# Animation prompts
	"animation_prompts": animation_prompts,

	# Output settings
	"fps": int(analysis["fps"]),
	"batch_name": "MusicDeforum",
	"filename_format": "{timestring}_{index}_{prompt}.png",
	"save_samples": True,
	"save_settings": True,
	"display_samples": True,

	# Video settings
	"skip_video_for_run_all": False,
	"make_gif": False,
	"delete_imgs": False
	}

	return preset


	# Initialize the generator
	generator = DeforumMusicGenerator()


	def process_audio(audio_file: Union[str,
	None],
	fps: int,
	width: int,
	height: int,
	steps: int,
	cfg_scale: float,
	sampler: str,
	seed: int,
	base_prompt: str,
	style: str,
	theme: str,
	selected_variants: List[str]) -> Tuple[str,
	str,
	Union[str,
	None]]:
	"""Main processing function with proper type annotations"""

	if not audio_file:
	return "❌ Please upload an audio file first.", "", None

	if not HAS_LIBROSA:
	return "❌ librosa is not installed. Please run: pip install librosa soundfile", "", None

	try:
	# Analyze the audio file
	status_msg = "🎵 Analyzing audio file...\n"
	analysis = generator.analyze_audio_file(audio_file, fps)

	duration = analysis["duration"]
	tempo = analysis["tempo"]
	total_frames = analysis["total_frames"]
	beat_count = len(analysis["beat_frames"])

	status_msg += f"✅ Audio Analysis Complete:\n"
	status_msg += f" Duration: {duration:.2f} seconds\n"
	status_msg += f" Tempo: {tempo:.1f} BPM\n"
	status_msg += f" Total Frames: {total_frames}\n"
	status_msg += f" Beats Detected: {beat_count}\n\n"

	# Prepare settings
	settings = {
	"width": width,
	"height": height,
	"fps": fps,
	"steps": steps,
	"cfg_scale": cfg_scale,
	"sampler": sampler,
	"seed": seed
	}

	# Generate presets for selected variants
	generated_presets = {}
	temp_files = []

	for variant_name in selected_variants:
	if variant_name not in generator.PRESET_VARIANTS:
	continue

	status_msg += f"🎨 Generating {variant_name} preset...\n"

	variant_settings = generator.PRESET_VARIANTS[variant_name]

	# Generate keyframes based on audio
	keyframes = generator.generate_keyframes(
	analysis, variant_settings)

	# Generate prompts
	prompts = generator.generate_prompts(
	analysis, base_prompt, style, theme,
	variant_settings["beats_per_section"]
	)

	# Create complete Deforum preset
	preset = generator.create_deforum_preset(
	analysis, settings, keyframes, prompts)
	generated_presets[variant_name] = preset

	# Save to temporary file
	temp_file = tempfile.NamedTemporaryFile(
	mode='w', suffix=f'_{variant_name}.json', delete=False)
	json.dump(preset, temp_file, indent=2, ensure_ascii=False)
	temp_file.close()
	temp_files.append((variant_name, temp_file.name))

	if not generated_presets:
	return "❌ No presets generated. Please select at least one variant.", "", None

	# Create ZIP file with all presets
	zip_file_path = tempfile.NamedTemporaryFile(
	suffix='.zip', delete=False)
	zip_file_path.close()

	with zipfile.ZipFile(zip_file_path.name, 'w', zipfile.ZIP_DEFLATED) as zip_file:
	for variant_name, file_path in temp_files:
	zip_file.write(
	file_path, f"deforum_preset_{variant_name}.json")

	# Create preview of generated presets
	preview_data = {}
	for variant_name, preset in generated_presets.items():
	preview_data[variant_name] = {
	"Total Frames": preset["max_frames"],
	"Dimensions": f"{preset['W']}x{preset['H']}",
	"Steps": preset["steps"],
	"CFG Scale": preset["scale"],
	"Sampler": preset["sampler"],
	"Prompt Count": len(preset["animation_prompts"]),
	"First Prompt": list(preset["animation_prompts"].values())[0][:100] + "..." if preset["animation_prompts"] else "None"
	}

	status_msg += f"✅ Successfully generated {len(generated_presets)} Deforum presets!\n\n"
	status_msg += "📥 Download the ZIP file and extract the JSON files.\n"
	status_msg += "🎬 In Deforum: Load Settings → Select JSON file → Generate!\n"

	preview_json = json.dumps(preview_data, indent=2)

	return status_msg, preview_json, zip_file_path.name

	except Exception as e:
	error_msg = f"❌ Error processing audio file:\n\n{str(e)}\n\n"
	error_msg += f"Traceback:\n{traceback.format_exc()}"
	return error_msg, "", None


	def create_interface():
	"""Create Gradio interface"""

	with gr.Blocks(
	title="Deforum Music Generator",
	theme=gr.themes.Soft(),
	css="""
	.main-container {
	max-width: 1200px;
	margin: 0 auto;
	}
	"""
	) as interface:

	gr.Markdown("""
	# 🎵 Deforum Music-to-Video Generator

	Transform your music into synchronized Deforum animation presets!

	Upload a music file and generate camera movements that sync to the beat and energy of your track.
	""")

	# Dependency status
	status_msgs = []
	if not HAS_LIBROSA:
	status_msgs.append(
	"❌ librosa missing - Install with: `pip install librosa soundfile`")
	if not HAS_WHISPER:
	status_msgs.append(
	"⚠️ whisper missing - Optional for lyrics: `pip install openai-whisper`")
	if not HAS_OPENAI:
	status_msgs.append(
	"⚠️ openai missing - Optional for GPT: `pip install openai`")

	if status_msgs:
	gr.Markdown("\n".join(status_msgs))
	else:
	gr.Markdown("✅ All dependencies loaded successfully!")

	with gr.Tab("🎵 Generate Presets"):

	with gr.Row():
	with gr.Column(scale=1):

	# Audio input
	audio_input = gr.Audio(
	label="🎵 Upload Music File",
	type="filepath"
	)

	# Video settings
	with gr.Group():
	gr.Markdown("### 📹 Video Settings")

	fps = gr.Slider(
	minimum=12, maximum=60, value=24, step=1,
	label="FPS (Frames Per Second)"
	)

	with gr.Row():
	width = gr.Slider(
	minimum=256, maximum=2048, value=1024, step=64,
	label="Width"
	)
	height = gr.Slider(
	minimum=256, maximum=1536, value=576, step=64,
	label="Height"
	)

	# Generation settings
	with gr.Group():
	gr.Markdown("### ⚙️ Generation Settings")

	steps = gr.Slider(
	minimum=10, maximum=100, value=30, step=1,
	label="Steps"
	)

	cfg_scale = gr.Slider(
	minimum=1.0, maximum=20.0, value=7.0, step=0.5,
	label="CFG Scale"
	)

	sampler = gr.Dropdown(
	choices=[
	"DPM++ 2M Karras",
	"Euler a",
	"DDIM",
	"LMS Karras",
	"DPM++ SDE Karras",
	"PLMS"
	],
	value="DPM++ 2M Karras",
	label="Sampler"
	)

	seed = gr.Number(
	value=-1,
	label="Seed (-1 for random)",
	precision=0
	)

	with gr.Column(scale=1):

	# Prompt settings
	with gr.Group():
	gr.Markdown("### 📝 Prompt Settings")

	base_prompt = gr.Textbox(
	value="abstract art, flowing shapes, vibrant colors, cinematic lighting",
	label="Base Prompt",
	lines=3,
	placeholder="Describe the visual style you want...")

	style = gr.Textbox(
	value="highly detailed, artistic, professional photography",
	label="Style Modifiers",
	lines=2,
	placeholder="Additional style keywords...")

	theme = gr.Textbox(
	value="",
	label="Theme (Optional)",
	lines=1,
	placeholder="e.g., cyberpunk, nature, space, fantasy...")

	# Variant selection
	with gr.Group():
	gr.Markdown("### 🎭 Preset Variants")

	variants = gr.CheckboxGroup(
	choices=["Calm", "Punchy", "Glitch"],
	value=["Calm", "Punchy"],
	label="Select Variants to Generate"
	)

	gr.Markdown("""
	- Calm: Subtle movements, smooth transitions
	- Punchy: Medium intensity, good for most music
	- Glitch: High intensity, chaotic movements
	""")

	# Generate button
	generate_btn = gr.Button(
	"🚀 Generate Deforum Presets",
	variant="primary",
	size="lg"
	)

	# Output section
	with gr.Row():
	with gr.Column():
	status_output = gr.Textbox(
	label="📊 Generation Status",
	lines=12,
	interactive=False,
	show_copy_button=True
	)

	with gr.Column():
	preview_output = gr.Code(
	label="👁️ Preset Preview",
	language="json",
	lines=12
	)

	# Download output
	download_output = gr.File(
	label="📥 Download Presets (ZIP file)",
	interactive=False
	)

	# Connect the generate button
	generate_btn.click(
	fn=process_audio,
	inputs=[
	audio_input,
	fps,
	width,
	height,
	steps,
	cfg_scale,
	sampler,
	seed,
	base_prompt,
	style,
	theme,
	variants],
	outputs=[
	status_output,
	preview_output,
	download_output],
	show_progress=True)

	with gr.Tab("📚 Usage Guide"):
	gr.Markdown("""
	## 🚀 How to Use

	### 1. Upload Music
	- Supported formats: MP3, WAV, FLAC, M4A
	- Recommended length: 1-5 minutes
	- The tool analyzes beats, tempo, and energy patterns

	### 2. Configure Settings

	Video Settings:
	- FPS: 24 is standard, 30+ for smoother motion
	- Dimensions: 1024x576 is a good balance of quality and speed

	Generation Settings:
	- Steps: 30-40 for good quality, 50+ for best results
	- CFG Scale: 7-10 for good prompt adherence
	- Sampler: DPM++ 2M Karras is reliable and fast

	Prompts:
	- Base Prompt: Your main visual concept
	- Style: Quality and aesthetic modifiers
	- Theme: Optional thematic elements

	### 3. Select Variants
	- Calm: Subtle camera movements, dreamy atmosphere
	- Punchy: Dynamic movements that follow the beat
	- Glitch: Intense, chaotic movements for electronic music

	### 4. Generate and Use
	1. Click "Generate Deforum Presets"
	2. Wait for processing (usually 1-2 minutes)
	3. Download the ZIP file
	4. Extract JSON files
	5. In Deforum: Load Settings → Select JSON → Generate!

	## 🎨 Tips for Better Results

	### Prompt Writing
	```
	Good: "cyberpunk cityscape, neon lights, rain, cinematic"
	Better: "cyberpunk cityscape, vibrant neon reflections, volumetric rain, cinematic lighting, highly detailed"
	```

	### Music Type Recommendations

	Electronic/EDM: Punchy or Glitch variants
	```
	Base: "abstract digital art, geometric patterns, vibrant colors"
	Style: "synthwave aesthetic, neon glow, high contrast"
	```

	Rock/Metal: Punchy or Glitch variants
	```
	Base: "dramatic landscape, storm clouds, lightning"
	Style: "dark atmosphere, intense lighting, cinematic"
	```

	Ambient/Classical: Calm variant
	```
	Base: "ethereal cloudscapes, flowing forms, soft light"
	Style: "painterly, dreamy, artistic, serene"
	```

	Hip-Hop: Punchy variant
	```
	Base: "urban environment, street art, dynamic angles"
	Style: "gritty texture, bold colors, street photography"
	```

	## 🔧 Troubleshooting

	"librosa not installed"
	```bash
	pip install librosa soundfile
	```

	"Audio analysis failed"
	- Check that FFmpeg is installed
	- Try a different audio format
	- Ensure file isn't corrupted

	"Preset won't load in Deforum"
	- Make sure you have the latest Deforum extension
	- Try loading the JSON in a text editor to check for errors
	- Verify all required fields are present

	Poor synchronization
	- Try different FPS settings
	- Ensure your music has clear beats
	- Consider the "Punchy" variant for better beat detection
	""")

	return interface


	# Main execution
	if __name__ == "__main__":
	print("🎵 Starting Deforum Music Generator...")
	print(f"Gradio version: {gr.__version__}")

	# Create and launch the interface
	app = create_interface()

	# Launch with corrected settings
	try:
	app.launch(
	share=True, # Set to False for local testing
	server_name="0.0.0.0", # Allow external connections
	server_port=7861,
	show_error=True,
	quiet=False,
	show_api=False # Disable API docs to avoid schema issues
	)
	except Exception as e:
	print(f"❌ Launch failed: {e}")
	print("Trying fallback launch configuration...")
	app.launch(
	share=False,
	inbrowser=True,
	show_error=True
	)