src/interpolation.py

"""
Interpolation Module
Handles latent space interpolation for smooth frame transitions.
"""

import torch
from diffusers import AutoencoderKL, StableDiffusionPipeline
from PIL import Image
from typing import List, Optional
import numpy as np


class LatentInterpolator:
    """
    Interpolates between keyframes in latent space for smooth animations.
    """
    
    def __init__(
        self,
        model_id: str = "Manojb/stable-diffusion-2-1-base",
        device: str = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu",
        torch_dtype: torch.dtype = torch.float16 if torch.cuda.is_available() else torch.float32
    ):
        """
        Initialize the interpolator.
        
        Args:
            model_id: Stable Diffusion model for VAE
            device: Device to run on
            torch_dtype: Data type for tensors
        """
        self.device = device
        self.torch_dtype = torch_dtype
        
        print(f"Loading VAE from: {model_id}")
        
        # Load VAE for encoding/decoding
        self.vae = AutoencoderKL.from_pretrained(
            model_id,
            subfolder="vae",
            torch_dtype=torch_dtype
        ).to(device)
        
        print("VAE loaded successfully!")
    
    def encode_image(self, image: Image.Image) -> torch.Tensor:
        """
        Encode PIL Image to latent representation.
        
        Args:
            image: PIL Image to encode
            
        Returns:
            Latent tensor
        """
        # Preprocess image
        image = image.resize((512, 512))
        image_np = np.array(image).astype(np.float32) / 255.0
        image_np = image_np * 2.0 - 1.0  # Normalize to [-1, 1]
        
        # Convert to tensor
        image_tensor = torch.from_numpy(image_np).permute(2, 0, 1).unsqueeze(0)
        image_tensor = image_tensor.to(device=self.device, dtype=self.torch_dtype)
        
        # Encode to latent space
        with torch.no_grad():
            latent = self.vae.encode(image_tensor).latent_dist.sample()
            latent = latent * 0.18215  # Scaling factor for SD
        
        return latent
    
    def decode_latent(self, latent: torch.Tensor) -> Image.Image:
        """
        Decode latent representation to PIL Image.
        
        Args:
            latent: Latent tensor
            
        Returns:
            PIL Image
        """
        # Decode from latent space
        with torch.no_grad():
            latent = latent / 0.18215
            image_tensor = self.vae.decode(latent).sample
        
        # Post-process
        image_tensor = (image_tensor / 2 + 0.5).clamp(0, 1)
        image_np = image_tensor.cpu().permute(0, 2, 3, 1).numpy()[0]
        image_np = (image_np * 255).astype(np.uint8)
        
        return Image.fromarray(image_np)
    
    def interpolate_latents(
        self,
        latent_start: torch.Tensor,
        latent_end: torch.Tensor,
        num_steps: int = 10,
        interpolation_type: str = "linear"
    ) -> List[torch.Tensor]:
        """
        Interpolate between two latent representations.
        
        Args:
            latent_start: Starting latent
            latent_end: Ending latent
            num_steps: Number of interpolation steps
            interpolation_type: Type of interpolation (linear, slerp)
            
        Returns:
            List of interpolated latents
        """
        interpolated = []
        
        for i in range(num_steps):
            alpha = i / (num_steps - 1) if num_steps > 1 else 0
            
            if interpolation_type == "linear":
                # Linear interpolation
                latent = (1 - alpha) * latent_start + alpha * latent_end
            elif interpolation_type == "slerp":
                # Spherical linear interpolation (better for latent spaces)
                latent = self._slerp(latent_start, latent_end, alpha)
            else:
                raise ValueError(f"Unknown interpolation type: {interpolation_type}")
            
            interpolated.append(latent)
        
        return interpolated
    
    def _slerp(
        self,
        v0: torch.Tensor,
        v1: torch.Tensor,
        t: float,
        eps: float = 1e-8
    ) -> torch.Tensor:
        """
        Spherical linear interpolation.
        
        Args:
            v0: Start vector
            v1: End vector
            t: Interpolation parameter [0, 1]
            eps: Small value to prevent division by zero
            
        Returns:
            Interpolated tensor
        """
        # Normalize vectors
        v0_norm = v0 / (torch.norm(v0) + eps)
        v1_norm = v1 / (torch.norm(v1) + eps)
        
        # Compute angle
        dot = (v0_norm * v1_norm).sum()
        dot = torch.clamp(dot, -1.0, 1.0)
        omega = torch.acos(dot)
        
        # Compute interpolation
        so = torch.sin(omega)
        if so < eps:
            # Vectors are nearly parallel, use linear interpolation
            return (1.0 - t) * v0 + t * v1
        
        return (torch.sin((1.0 - t) * omega) / so) * v0 + (torch.sin(t * omega) / so) * v1
    
    def generate_interpolated_sequence(
        self,
        keyframes: List[Image.Image],
        frames_between: int = 4,
        interpolation_type: str = "slerp"
    ) -> List[Image.Image]:
        """
        Generate smooth sequence by interpolating between keyframes.
        
        Args:
            keyframes: List of keyframe images
            frames_between: Number of frames to generate between each keyframe
            interpolation_type: Interpolation method
            
        Returns:
            List of all frames (keyframes + interpolated)
        """
        if len(keyframes) < 2:
            return keyframes
        
        all_frames = []
        
        for i in range(len(keyframes) - 1):
            # Encode keyframes to latent space
            latent_start = self.encode_image(keyframes[i])
            latent_end = self.encode_image(keyframes[i + 1])
            
            # Interpolate
            interpolated_latents = self.interpolate_latents(
                latent_start,
                latent_end,
                num_steps=frames_between + 2,  # +2 to include start and end
                interpolation_type=interpolation_type
            )
            
            # Decode all except the last (avoid duplicates)
            for latent in interpolated_latents[:-1]:
                frame = self.decode_latent(latent)
                all_frames.append(frame)
        
        # Add final keyframe
        all_frames.append(keyframes[-1])
        
        return all_frames


if __name__ == "__main__":
    print("Latent interpolation module loaded.")
    print("Use with keyframe images to generate smooth interpolations.")