update

services/compress/scene_detector.py

@@ -1,60 +1,25 @@
-"""
-Scene Detection Module
-
-This module handles scene detection and splitting for video processing.
-It supports multiple detection modes including adaptive, time-based, and forced splitting.
-"""
-
-import json
 import os
-from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Any
-
+import json
+import subprocess
 from utils.fast_scene_detect import adaptive_scene_detection_check
 from utils.split_video_into_scenes import split_video_into_scenes
 from utils.video_utils import get_keyframes
 
-
-# ============================================================================
-# Constants
-# ============================================================================
-
-# Default configuration values
-DEFAULT_CONFIG = {
-    'directories': {
-        'temp_dir': './videos/temp_scenes',
-        'output_dir': './output'
-    },
-    'video_processing': {
-        'SHORT_VIDEO_THRESHOLD': 20,
-        'target_vmaf': 93.0
-    },
-    'scene_detection': {
-        'enable_time_based_fallback': True,
-        'time_based_scene_duration': 60
-    }
-}
-
-
-# ============================================================================
-# Main Function
-# ============================================================================
-
-def scene_detection(video_metadata: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]:
+def scene_detection(video_metadata):
     """
     Part 2: Scene detection based on video length using metadata from part 1.
     
     Args:
-        video_metadata: Metadata from part1_pre_processing containing:
-                       - 'path': video file path
-                       - 'codec': current video codec
-                       - 'original_codec': original codec (if reencoded)
-                       - 'target_codec': target codec for encoding
-                       - 'duration': video duration in seconds
-                       - 'was_reencoded': boolean
-                       - 'target_vmaf': target VMAF score
-                       - 'target_quality': target quality level
-                       - 'processing_info': processing details
+        video_metadata (dict): Metadata from part1_pre_processing containing:
+                              - 'path': video file path
+                              - 'codec': current video codec
+                              - 'original_codec': original codec (if reencoded)
+                              - 'target_codec': target codec for encoding (NEW)
+                              - 'duration': video duration in seconds
+                              - 'was_reencoded': boolean
+                              - 'target_vmaf': target VMAF score
+                              - 'target_quality': target quality level
+                              - 'processing_info': processing details
         
     Returns:
         list: List of scene metadata dictionaries, each containing:
@@ -66,14 +31,34 @@ def scene_detection(video_metadata: Dict[str, Any]) -> Optional[List[Dict[str, A
               - 'original_video_metadata': complete metadata from part 1
     """
     # Load configuration
-    config = _load_configuration()
+    try:
+        with open('services/compress/config.json', 'r') as f:
+            config = json.load(f)
+        print("✅ Configuration loaded successfully")
+    except FileNotFoundError:
+        print("⚠️ Config file not found, using default configuration")
+        config = {
+            'directories': {
+                'temp_dir': './videos/temp_scenes',
+                'output_dir': './output'
+            },
+            'video_processing': {
+                'SHORT_VIDEO_THRESHOLD': 20,
+                'target_vmaf': 93.0
+            },
+            'scene_detection': {
+                'enable_time_based_fallback': True,
+                'time_based_scene_duration': 60
+            }
+        }
     
     # Extract metadata
     video_path = video_metadata['path']
     duration = video_metadata['duration']
     codec = video_metadata['codec']
     original_codec = video_metadata.get('original_codec', codec)
-    target_codec = video_metadata.get('target_codec', 'auto')
+    target_codec = video_metadata.get('target_codec', 'auto') 
+    print("***********target_codec", target_codec) # ✅ NEW: Extract target codec
     was_reencoded = video_metadata.get('was_reencoded', False)
     target_vmaf = video_metadata.get('target_vmaf', 93.0)
     target_quality = video_metadata.get('target_quality', 'Medium')
@@ -81,118 +66,110 @@ def scene_detection(video_metadata: Dict[str, Any]) -> Optional[List[Dict[str, A
     print(f"🎬 Processing video: {os.path.basename(video_path)}")
     print(f"   ⏱️ Duration: {duration:.1f}s, Codec: {codec}, Reencoded: {was_reencoded}")
     print(f"   🎯 Target: {target_quality} (VMAF: {target_vmaf})")
-    print(f"   🎥 Codec flow: {original_codec} → {codec} → {target_codec}")
+    print(f"   🎥 Codec flow: {original_codec} → {codec} → {target_codec}")  # ✅ NEW: Show codec flow
     
     # Get processing configuration
     video_processing_config = config.get('video_processing', {})
     short_video_threshold = video_processing_config.get('SHORT_VIDEO_THRESHOLD', 20)
+    # Get directories from config only
     temp_dir = config.get('directories', {}).get('temp_dir', './videos/temp_scenes')
     
-    # Handle short videos
     if duration <= short_video_threshold:
         print(f"📏 Video is shorter than {short_video_threshold}s. Treating as a single scene.")
-        return _create_single_scene_metadata(video_metadata, duration)
-    
-    # Get scene detection configuration
+        return [{
+            'path': video_metadata['path'],
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': duration,
+            'duration': duration,
+            'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
+        }]
     scene_config = config.get('scene_detection', {})
     detection_mode = scene_config.get('mode', 'adaptive').lower()
     
     print(f"   🔍 Scene detection mode: {detection_mode}")
     
-    # Execute appropriate detection method
     if detection_mode == 'force_time_based':
-        return _execute_force_time_based_splitting(
-            video_path, temp_dir, scene_config, duration, video_metadata
-        )
+        # Force time-based splitting regardless of content
+        force_duration = scene_config.get('force_time_based_duration', 10)
+        print(f"   ⏰ Forcing time-based splitting with {force_duration}s segments")
+        return force_time_based_splitting(video_path, temp_dir, force_duration, duration, video_metadata)
+    
     elif detection_mode == 'adaptive':
-        return _execute_adaptive_scene_detection(
-            video_path, temp_dir, scene_config, duration, video_metadata, config
+        # Original adaptive scene detection logic
+        print(f"   🔍 Running adaptive scene detection...")
+        has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
+            video_path, duration, logging_enabled=True
         )
+
+        if has_multiple_scenes:
+            print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
+            scene_files, _ = split_video_into_scenes(video_path, temp_dir)
+            
+            if scene_files and len(scene_files) > 1:
+                print(f"   📄 Created {len(scene_files)} scene files")
+                return create_scene_metadata_from_files(scene_files, video_metadata, config)
+            else:
+                print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+        else:
+            print(f"   📏 Single scene detected.")
+            enable_fallback = scene_config.get('enable_time_based_fallback', False)
+            
+            if enable_fallback:
+                print(f"   🔄 Time-based fallback enabled. Attempting time-based scene splitting...")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+            else:
+                print(f"   ✅ Using single scene (time-based fallback disabled)")
+                return [{
+                    'path': video_path,
+                    'scene_number': 1,
+                    'start_time': 0.0,
+                    'end_time': duration,
+                    'duration': duration,
+                    'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
+                }]
+    
     else:
         print(f"   ❌ Unknown scene detection mode: {detection_mode}. Falling back to adaptive.")
-        return _execute_adaptive_scene_detection(
-            video_path, temp_dir, scene_config, duration, video_metadata, config
+        # Fallback to adaptive mode
+        print(f"   🔍 Running adaptive scene detection...")
+        has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
+            video_path, duration, logging_enabled=True
         )
 
-
-# ============================================================================
-# Scene Detection Methods
-# ============================================================================
-
-def _execute_force_time_based_splitting(
-    video_path: str, 
-    temp_dir: str, 
-    scene_config: Dict[str, Any], 
-    duration: float, 
-    video_metadata: Dict[str, Any]
-) -> List[Dict[str, Any]]:
-    """Execute forced time-based scene splitting."""
-    force_duration = scene_config.get('force_time_based_duration', 10)
-    print(f"   ⏰ Forcing time-based splitting with {force_duration}s segments")
-    return force_time_based_splitting(
-        video_path, temp_dir, force_duration, duration, video_metadata
-    )
-
-
-def _execute_adaptive_scene_detection(
-    video_path: str, 
-    temp_dir: str, 
-    scene_config: Dict[str, Any], 
-    duration: float, 
-    video_metadata: Dict[str, Any], 
-    config: Dict[str, Any]
-) -> List[Dict[str, Any]]:
-    """Execute adaptive scene detection with fallback options."""
-    print(f"   🔍 Running adaptive scene detection...")
-    
-    has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
-        video_path, duration, logging_enabled=True
-    )
-
-    if has_multiple_scenes:
-        print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
-        scene_files, _ = split_video_into_scenes(video_path, temp_dir)
-        
-        if scene_files and len(scene_files) > 1:
-            print(f"   📄 Created {len(scene_files)} scene files")
-            return create_scene_metadata_from_files(scene_files, video_metadata, config)
-        else:
-            print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
-            return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
-    else:
-        print(f"   📏 Single scene detected.")
-        enable_fallback = scene_config.get('enable_time_based_fallback', False)
-        
-        if enable_fallback:
-            print(f"   🔄 Time-based fallback enabled. Attempting time-based scene splitting...")
-            return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+        if has_multiple_scenes:
+            print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
+            scene_files, _ = split_video_into_scenes(video_path, temp_dir)
+            
+            if scene_files and len(scene_files) > 1:
+                print(f"   📄 Created {len(scene_files)} scene files")
+                return create_scene_metadata_from_files(scene_files, video_metadata, config)
+            else:
+                print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
         else:
-            print(f"   ✅ Using single scene (time-based fallback disabled)")
-            return _create_single_scene_metadata(video_metadata, duration)
-
-
-# ============================================================================
-# Time-based Splitting Functions
-# ============================================================================
-
-def force_time_based_splitting(
-    video_path: str, 
-    temp_dir: str, 
-    segment_duration: float, 
-    total_duration: float, 
-    original_metadata: Dict[str, Any], 
-    logging_enabled: bool = True
-) -> List[Dict[str, Any]]:
+            print(f"   📏 Single scene detected. Using single scene.")
+            return [{
+                'path': video_path,
+                'scene_number': 1,
+                'start_time': 0.0,
+                'end_time': duration,
+                'duration': duration,
+                'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
+            }]
+
+def force_time_based_splitting(video_path, temp_dir, segment_duration, total_duration, original_metadata, logging_enabled=True):
     """
     Force time-based splitting with keyframe-aligned boundaries to prevent stuttering.
     
     Args:
-        video_path: Path to the video file
-        temp_dir: Temporary directory for scene files
-        segment_duration: Target duration for each segment in seconds
-        total_duration: Total video duration in seconds
-        original_metadata: Complete metadata from Part 1
-        logging_enabled: Enable detailed logging
+        video_path (str): Path to the video file
+        temp_dir (str): Temporary directory for scene files
+        segment_duration (float): Target duration for each segment in seconds
+        total_duration (float): Total video duration in seconds
+        original_metadata (dict): Complete metadata from Part 1
+        logging_enabled (bool): Enable detailed logging
     
     Returns:
         list: List of scene metadata dictionaries
@@ -204,181 +181,434 @@ def force_time_based_splitting(
         print(f"   ✂️ Segment duration: {segment_duration}s")
     
     # Get keyframes for precise cutting
-    keyframes = _get_video_keyframes(video_path, logging_enabled)
+    try:
+        keyframes = get_keyframes(video_path)
+        if logging_enabled:
+            print(f"   🔑 Found {len(keyframes)} keyframes for alignment")
+    except Exception as e:
+        # Try alternative import path
+        try:
+            keyframes = get_keyframes(video_path)
+            if logging_enabled:
+                print(f"   🔑 Found {len(keyframes)} keyframes for alignment")
+        except Exception as e2:
+            keyframes = []
+            if logging_enabled:
+                print(f"   ⚠️ Could not get keyframes: {e}")
+                print(f"   📐 Using time-based boundaries (may cause stuttering)")
     
     # Calculate scene boundaries, preferring keyframe alignment
-    scene_boundaries = _calculate_scene_boundaries(
-        total_duration, segment_duration, keyframes, logging_enabled
-    )
+    scene_boundaries = []
+    current_start = 0
+    
+    while current_start < total_duration:
+        target_end = min(current_start + segment_duration, total_duration)
+        
+        # If we have keyframes, find the closest keyframe to target_end
+        if keyframes and target_end < total_duration:
+            # Find keyframe closest to target_end
+            closest_keyframe = min(keyframes, key=lambda kf: abs(kf - target_end))
+            
+            # Only use keyframe if it's reasonably close (within 2 seconds)
+            if abs(closest_keyframe - target_end) <= 2.0 and closest_keyframe > current_start:
+                scene_end = closest_keyframe
+            else:
+                scene_end = target_end
+        else:
+            scene_end = target_end
+        
+        # Handle very short final segments (< 2 seconds) by merging with previous scene
+        remaining_duration = total_duration - scene_end
+        if remaining_duration > 0 and remaining_duration < 2.0 and scene_boundaries:
+            # Extend current scene to end of video instead of creating tiny final scene
+            scene_end = total_duration
+            if logging_enabled:
+                print(f"   🔧 Merging final {remaining_duration:.1f}s with current scene to avoid stuttering")
+        
+        scene_boundaries.append((current_start, scene_end))
+        current_start = scene_end
+    
+    if logging_enabled:
+        print(f"   📊 Will create {len(scene_boundaries)} segments:")
+        for i, (start, end) in enumerate(scene_boundaries):
+            print(f"      Segment {i+1}: {start:.1f}s - {end:.1f}s (duration: {end-start:.1f}s)")
     
-    # Create scene metadata
-    scenes = []
-    for i, (start_time, end_time) in enumerate(scene_boundaries):
+    # If only one scene, return original video
+    if len(scene_boundaries) <= 1:
+        if logging_enabled:
+            print(f"   📏 Video shorter than segment duration, using as single scene")
+        return [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
+    
+    # Create output directory
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    # Choose scene file extension based on source codec
+    source_codec = original_metadata.get('codec', 'h264').lower()
+    if source_codec in ['ffv1', 'prores', 'dnxhd']:
+        scene_extension = '.mkv'  # Use MKV for lossless codecs
+    elif source_codec in ['av1', 'vp9']:
+        scene_extension = '.webm'  # Use WebM for modern codecs
+    else:
+        scene_extension = '.mp4'  # Default to MP4 for compatibility
+    
+    if logging_enabled:
+        print(f"   📦 Using {scene_extension} container for {source_codec} codec")
+    
+    # Split video and create metadata
+    scene_metadata_list = []
+    
+    for i, (start_time_sec, end_time_sec) in enumerate(scene_boundaries):
         scene_number = i + 1
-        scene_duration = end_time - start_time
+        scene_filename = f"scene_{scene_number:03d}_time_{start_time_sec:.0f}s{scene_extension}"
+        scene_path = os.path.join(temp_dir, scene_filename)
+        
+        duration_scene = end_time_sec - start_time_sec
+        
+        # FFmpeg command with precise cutting to prevent frame alignment issues
+        # Use input seeking for efficiency and output seeking for precision
+        cmd = [
+            'ffmpeg', '-y',
+            '-ss', str(start_time_sec),     # Input seeking for efficiency
+            '-i', video_path,
+            '-t', str(duration_scene),      # Precise duration
+            '-avoid_negative_ts', 'make_zero',
+            '-fflags', '+genpts',           # Generate presentation timestamps
+            '-map', '0:v:0',                # Map video stream explicitly
+            '-map', '0:a?',                 # Map audio if exists (optional)
+        ]
+        
+        # Add codec-specific format options before encoding
+        format_options = []
+        if source_codec == 'ffv1':
+            format_options.extend(['-f', 'matroska'])
+        elif source_codec in ['av1', 'vp9']:
+            format_options.extend(['-f', 'webm'])
+        
+        # Always use fast re-encoding to ensure clean cuts and prevent stuttering
+        # Stream copy can cause artifacts at scene boundaries even with keyframe alignment
+        cmd.extend([
+            '-c:v', 'libx264',
+            '-preset', 'ultrafast',
+            '-crf', '18',
+            '-c:a', 'copy'
+        ])
+        cmd.extend(format_options)  # Add format options
+        cmd.append(scene_path)      # Add output file path
         
         if logging_enabled:
-            print(f"   Scene {scene_number}: {start_time:.1f}s - {end_time:.1f}s (duration: {scene_duration:.1f}s)")
+            print(f"   ✂️ Creating segment {scene_number}: {scene_filename}")
         
-        scenes.append({
-            'path': video_path,  # Original video path for time-based scenes
-            'scene_number': scene_number,
-            'start_time': start_time,
-            'end_time': end_time,
-            'duration': scene_duration,
-            'original_video_metadata': original_metadata,
-            'splitting_method': 'force_time_based',
-            'segment_duration': segment_duration
-        })
+        try:
+            # Execute FFmpeg command
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+            
+            if result.returncode == 0:
+                # Verify file exists and has content
+                if os.path.exists(scene_path) and os.path.getsize(scene_path) > 0:
+                    file_size_mb = os.path.getsize(scene_path) / (1024 * 1024)
+                    
+                    scene_metadata = {
+                        'path': scene_path,
+                        'scene_number': scene_number,
+                        'start_time': start_time_sec,
+                        'end_time': end_time_sec,
+                        'duration': duration_scene,
+                        'scene_codec': source_codec,
+                        'scene_container': scene_extension[1:],  # Remove the dot
+                        'file_size_mb': file_size_mb,
+                        'scene_detection_method': 'force_time_based',
+                        'original_video_metadata': original_metadata
+                    }
+                    scene_metadata_list.append(scene_metadata)
+                    
+                    if logging_enabled:
+                        print(f"      ✅ Created: {scene_filename} ({file_size_mb:.1f} MB)")
+                else:
+                    if logging_enabled:
+                        print(f"      ❌ Failed: {scene_filename} (file not created or empty)")
+            else:
+                if logging_enabled:
+                    print(f"      ❌ FFmpeg failed for {scene_filename}: {result.stderr}")
+                
+        except subprocess.TimeoutExpired:
+            if logging_enabled:
+                print(f"      ❌ Timeout creating {scene_filename}")
+        except Exception as e:
+            if logging_enabled:
+                print(f"      ❌ Error creating {scene_filename}: {e}")
+    
+    if not scene_metadata_list:
+        if logging_enabled:
+            print(f"   ❌ All segment creation failed, using original video as single scene")
+        return [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
     
     if logging_enabled:
-        print(f"   ✅ Created {len(scenes)} time-based scenes")
+        print(f"   ✅ Successfully created {len(scene_metadata_list)} time-based segments")
     
-    return scenes
-
+    return scene_metadata_list
 
-def time_based_splitting(
-    video_path: str, 
-    temp_dir: str, 
-    config: Dict[str, Any], 
-    duration: float, 
-    video_metadata: Dict[str, Any]
-) -> List[Dict[str, Any]]:
+def time_based_splitting(video_path, temp_dir, config, duration, original_metadata, logging_enabled=True):
     """
-    Time-based scene splitting using configuration settings.
+    Split video into scenes and return metadata for each scene.
     
     Args:
-        video_path: Path to the video file
-        temp_dir: Temporary directory for scene files
-        config: Configuration dictionary
-        duration: Video duration in seconds
-        video_metadata: Complete metadata from Part 1
-        
+        video_path (str): Path to the video file
+        temp_dir (str): Temporary directory for scene files
+        config (dict): Configuration dictionary
+        duration (float): Video duration in seconds
+        original_metadata (dict): Complete metadata from Part 1 including codec info
+        logging_enabled (bool): Enable detailed logging
+    
     Returns:
         list: List of scene metadata dictionaries
     """
-    scene_config = config.get('scene_detection', {})
-    time_based_duration = scene_config.get('time_based_scene_duration', 90)
-    
-    print(f"   ⏰ Time-based splitting with {time_based_duration}s segments")
+    if logging_enabled:
+        print(f"\n--- Running time-based scene splitting with metadata ---")
+        # ✅ NEW: Log codec information for scene splitting
+        print(f"   🎥 Source codec: {original_metadata.get('codec', 'unknown')}")
+        print(f"   🎯 Target codec: {original_metadata.get('target_codec', 'auto')}")
     
-    return force_time_based_splitting(
-        video_path, temp_dir, time_based_duration, duration, video_metadata
-    )
-
-
-# ============================================================================
-# Helper Functions
-# ============================================================================
-
-def _load_configuration() -> Dict[str, Any]:
-    """Load configuration from config.json or use defaults."""
     try:
-        with open('services/compress/config.json', 'r') as f:
-            config = json.load(f)
-        print("✅ Configuration loaded successfully")
-        return config
-    except FileNotFoundError:
-        print("⚠️ Config file not found, using default configuration")
-        return DEFAULT_CONFIG
-
-
-def _create_single_scene_metadata(
-    video_metadata: Dict[str, Any], 
-    duration: float
-) -> List[Dict[str, Any]]:
-    """Create metadata for a single scene video."""
-    return [{
-        'path': video_metadata['path'],
-        'scene_number': 1,
-        'start_time': 0.0,
-        'end_time': duration,
-        'duration': duration,
-        'original_video_metadata': video_metadata
-    }]
-
-
-def _get_video_keyframes(video_path: str, logging_enabled: bool) -> List[float]:
-    """Get video keyframes for precise scene cutting."""
-    try:
-        keyframes = get_keyframes(video_path)
+        total_duration = duration
         if logging_enabled:
-            print(f"   🔑 Found {len(keyframes)} keyframes for alignment")
-        return keyframes
-    except Exception as e:
+            print(f"Using provided duration: {total_duration:.1f}s")
+        
+        if not total_duration:
+            return [{
+                'path': video_path,
+                'scene_number': 1,
+                'start_time': 0.0,
+                'end_time': duration,
+                'duration': duration,
+                'original_video_metadata': original_metadata
+            }]
+        
+        # Get scene duration from config
+        scene_config = config.get('scene_detection', {})
+        scene_duration = scene_config.get('time_based_scene_duration', None)
+        
+        # Auto-calculate scene duration if not specified
+        if scene_duration is None:
+            if total_duration <= 120:
+                scene_duration = 60
+            elif total_duration <= 600:
+                scene_duration = 120
+            elif total_duration <= 1800:
+                scene_duration = 300
+            else:
+                scene_duration = 600
+            
+            if logging_enabled:
+                print(f"Auto-calculated scene duration: {scene_duration}s for {total_duration:.1f}s video")
+        
+        # Calculate scene boundaries
+        scene_boundaries = []
+        current_start = 0
+        
+        while current_start < total_duration:
+            scene_end = min(current_start + scene_duration, total_duration)
+            scene_boundaries.append((current_start, scene_end))
+            current_start = scene_end
+        
         if logging_enabled:
-            print(f"   ⚠️ Could not get keyframes: {e}")
-            print(f"   📐 Using time-based boundaries (may cause stuttering)")
-        return []
-
-
-def _calculate_scene_boundaries(
-    total_duration: float, 
-    segment_duration: float, 
-    keyframes: List[float], 
-    logging_enabled: bool
-) -> List[Tuple[float, float]]:
-    """Calculate scene boundaries with keyframe alignment when possible."""
-    boundaries = []
-    current_time = 0.0
-    
-    while current_time < total_duration:
-        end_time = min(current_time + segment_duration, total_duration)
+            print(f"Calculated {len(scene_boundaries)} time-based scenes:")
+            for i, (start, end) in enumerate(scene_boundaries):
+                print(f"  Scene {i+1}: {start:.1f}s - {end:.1f}s (duration: {end-start:.1f}s)")
+        
+        # If only one scene, return original video
+        if len(scene_boundaries) <= 1:
+            return [{
+                'path': video_path,
+                'scene_number': 1,
+                'start_time': 0.0,
+                'end_time': total_duration,
+                'duration': total_duration,
+                'original_video_metadata': original_metadata
+            }]
         
-        # Try to align with nearest keyframe
-        if keyframes:
-            nearest_keyframe = _find_nearest_keyframe(current_time, keyframes)
-            if nearest_keyframe and nearest_keyframe > current_time and nearest_keyframe < end_time:
-                end_time = nearest_keyframe
+        # Create output directory
+        os.makedirs(temp_dir, exist_ok=True)
+        
+        # ✅ IMPROVED: Choose scene file extension based on source codec
+        source_codec = original_metadata.get('codec', 'h264').lower()
+        if source_codec in ['ffv1', 'prores', 'dnxhd']:
+            scene_extension = '.mkv'  # Use MKV for lossless codecs
+        elif source_codec in ['av1', 'vp9']:
+            scene_extension = '.webm'  # Use WebM for modern codecs
+        else:
+            scene_extension = '.mp4'  # Default to MP4 for compatibility
+        
+        if logging_enabled:
+            print(f"   📦 Using {scene_extension} container for {source_codec} codec")
+        
+        # Split video and create metadata
+        scene_metadata_list = []
+        
+        for i, (start_time_sec, end_time_sec) in enumerate(scene_boundaries):
+            scene_number = i + 1
+            scene_filename = f"scene_{scene_number:03d}{scene_extension}"
+            scene_path = os.path.join(temp_dir, scene_filename)
+            
+            duration_scene = end_time_sec - start_time_sec
+            
+            # ✅ IMPROVED: FFmpeg command with codec-aware copy settings
+            cmd = [
+                'ffmpeg', '-y',
+                '-ss', str(start_time_sec),
+                '-i', video_path,
+                '-t', str(duration_scene),
+                '-c', 'copy',  # Stream copy for speed
+                '-map', '0',   # Copy all streams
+                '-avoid_negative_ts', 'make_zero',
+                scene_path
+            ]
+            
+            # ✅ NEW: Add codec-specific options if needed
+            if source_codec == 'ffv1':
+                # For FFV1, ensure proper MKV muxing
+                cmd.extend(['-f', 'matroska'])
+            elif source_codec in ['av1', 'vp9']:
+                # For modern codecs, ensure proper WebM muxing
+                cmd.extend(['-f', 'webm'])
+            
+            if logging_enabled:
+                print(f"Extracting scene {scene_number}/{len(scene_boundaries)}: {start_time_sec:.1f}s-{end_time_sec:.1f}s")
+            
+            try:
+                result = subprocess.run(
+                    cmd, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE, 
+                    text=True, 
+                    check=True
+                )
+                
+                if os.path.exists(scene_path) and os.path.getsize(scene_path) > 0:
+                    # Create scene metadata
+                    scene_metadata = {
+                        'path': scene_path,
+                        'scene_number': scene_number,
+                        'start_time': start_time_sec,
+                        'end_time': end_time_sec,
+                        'duration': duration_scene,
+                        'original_video_metadata': original_metadata,  # ✅ Complete metadata including codec info
+                        'file_size_mb': os.path.getsize(scene_path) / (1024 * 1024),
+                        'scene_codec': source_codec,  # ✅ NEW: Track scene codec
+                        'scene_container': scene_extension[1:]  # ✅ NEW: Track container format
+                    }
+                    scene_metadata_list.append(scene_metadata)
+                    
+                    if logging_enabled:
+                        print(f"  ✅ Created: {scene_filename} ({scene_metadata['file_size_mb']:.1f} MB)")
+                else:
+                    if logging_enabled:
+                        print(f"  ❌ Failed: {scene_filename}")
+                        
+            except subprocess.CalledProcessError as e:
                 if logging_enabled:
-                    print(f"      Aligned scene boundary with keyframe at {end_time:.1f}s")
+                    print(f"  ❌ FFmpeg error for scene {scene_number}: {e.stderr}")
+                continue
+            except Exception as e:
+                if logging_enabled:
+                    print(f"  ❌ Unexpected error for scene {scene_number}: {e}")
+                continue
         
-        boundaries.append((current_time, end_time))
-        current_time = end_time
-    
-    return boundaries
-
-
-def _find_nearest_keyframe(current_time: float, keyframes: List[float]) -> Optional[float]:
-    """Find the nearest keyframe after the current time."""
-    for keyframe in keyframes:
-        if keyframe > current_time:
-            return keyframe
-    return None
-
-
-def create_scene_metadata_from_files(
-    scene_files: List[str], 
-    video_metadata: Dict[str, Any], 
-    config: Dict[str, Any]
-) -> List[Dict[str, Any]]:
+        if logging_enabled:
+            print(f"Time-based splitting completed: {len(scene_metadata_list)} scenes created")
+            if scene_metadata_list:
+                total_size = sum(scene['file_size_mb'] for scene in scene_metadata_list)
+                print(f"   📊 Total scenes size: {total_size:.1f} MB")
+        
+        return scene_metadata_list if scene_metadata_list else [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
+        
+    except Exception as e:
+        if logging_enabled:
+            print(f"Time-based splitting failed: {e}")
+        return [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': duration,
+            'duration': duration,
+            'original_video_metadata': original_metadata
+        }]
+
+def create_scene_metadata_from_files(scene_files, original_metadata, config):
     """
-    Create scene metadata from split scene files.
+    Create metadata for scenes created by PySceneDetect.
+    Note: PySceneDetect doesn't give us exact timing, so we estimate.
     
     Args:
-        scene_files: List of scene file paths
-        video_metadata: Original video metadata
-        config: Configuration dictionary
-        
+        scene_files (list): List of scene file paths
+        original_metadata (dict): Complete metadata from Part 1 including codec info
+        config (dict): Configuration dictionary
+    
     Returns:
         list: List of scene metadata dictionaries
     """
-    scenes = []
+    scene_metadata_list = []
+    total_duration = original_metadata['duration']
+    source_codec = original_metadata.get('codec', 'unknown')
+    
+    if not scene_files:
+        return [{
+            'path': original_metadata['path'],
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
+    
+    # Estimate timing based on file count (not perfect, but reasonable)
+    estimated_scene_duration = total_duration / len(scene_files)
+    
+    print(f"   🎥 Creating metadata for {len(scene_files)} scenes from {source_codec} source")
     
     for i, scene_file in enumerate(scene_files):
         scene_number = i + 1
+        start_time = i * estimated_scene_duration
+        end_time = min((i + 1) * estimated_scene_duration, total_duration)
+        
+        # Adjust last scene to cover remaining time
+        if i == len(scene_files) - 1:
+            end_time = total_duration
         
-        # Get scene duration (you might want to implement this)
-        scene_duration = 0.0  # Placeholder
+        # ✅ NEW: Extract scene file extension for container tracking
+        scene_extension = os.path.splitext(scene_file)[1][1:] if os.path.splitext(scene_file)[1] else 'unknown'
         
-        scenes.append({
+        scene_metadata = {
             'path': scene_file,
             'scene_number': scene_number,
-            'start_time': 0.0,  # Placeholder
-            'end_time': scene_duration,
-            'duration': scene_duration,
-            'original_video_metadata': video_metadata,
-            'splitting_method': 'adaptive_detection'
-        })
-    
-    return scenes
+            'start_time': start_time,
+            'end_time': end_time,
+            'duration': end_time - start_time,
+            'original_video_metadata': original_metadata,  # ✅ Complete metadata including codec info
+            'file_size_mb': os.path.getsize(scene_file) / (1024 * 1024) if os.path.exists(scene_file) else 0,
+            'scene_codec': source_codec,  # ✅ NEW: Track scene codec
+            'scene_container': scene_extension  # ✅ NEW: Track container format
+        }
+        scene_metadata_list.append(scene_metadata)
+    
+    return scene_metadata_list

services/compress/validator_merger.py

@@ -1,55 +1,16 @@
-"""
-Validation and Merging Module
-
-This module handles the final stage of video processing including:
-- Scene validation and quality checks
-- Video merging from encoded scenes
-- Final VMAF calculation
-- Comprehensive report generation
-"""
-
 import os
 import json
 import time
-import signal
 from datetime import datetime
-from typing import Dict, List, Any, Optional, Tuple
-
 from utils.calculate_vmaf_adv import calculate_vmaf_advanced
 from utils.merge_videos import merge_videos
+import signal
 
-
-# ============================================================================
-# Constants
-# ============================================================================
-
-# Timeout settings
-MERGE_TIMEOUT_SECONDS = 300  # 5 minutes
-SIGNAL_TIMEOUT_SECONDS = 10
-
-
-# ============================================================================
-# Main Function
-# ============================================================================
-
-def validation_and_merging(
-    original_video_path: str, 
-    encoded_scenes_data: List[Dict[str, Any]], 
-    config: Dict[str, Any], 
-    logging_enabled: bool = True
-) -> Tuple[Optional[str], Optional[float], Optional[Dict[str, Any]]]:
+def validation_and_merging(original_video_path, encoded_scenes_data, config, logging_enabled=True):
     """
     Part 4: Validation and merging of encoded scenes with existing VMAF data.
-    
-    Args:
-        original_video_path: Path to the original video file
-        encoded_scenes_data: List of scene data dictionaries from Part 3
-        config: Configuration dictionary
-        logging_enabled: Whether to enable detailed logging
-        
-    Returns:
-        tuple: (final_video_path, final_vmaf, comprehensive_report)
     """
+    logging_enabled=True  # Ensure logging is enabled for this part
     if logging_enabled:
         print(f"\n🔗 --- Part 4: Validation and Merging ---")
         print(f"   🎬 Processing {len(encoded_scenes_data)} scenes with VMAF data")
@@ -73,81 +34,20 @@ def validation_and_merging(
         print(f"   📁 Output directory: {output_dir}")
         print(f"   📁 Temp directory: {temp_dir}")
     
-    # Create output directory with error handling
-    if not _create_output_directory(output_dir, logging_enabled):
-        return None, None, None
-    
-    # Initialize processing start time
-    part5_start_time = time.time()
-    
-    # Validate encoded scenes
-    successful_scenes, failed_scenes, temp_files_to_cleanup = _validate_encoded_scenes(
-        encoded_scenes_data, logging_enabled
-    )
-    
-    if not successful_scenes:
-        if logging_enabled:
-            print("❌ No valid scenes found for merging")
-        return None, None, None
-    
-    # Merge video scenes
-    final_output_path, merge_time, merged_size = _merge_video_scenes(
-        successful_scenes, output_dir, original_video_name, logging_enabled
-    )
-    
-    if not final_output_path:
-        return None, None, None
-    
-    # Clean up temporary files if enabled
-    if cleanup_temp_files:
-        _cleanup_temp_files(temp_files_to_cleanup, successful_scenes, logging_enabled)
-    
-    # Calculate final VMAF
-    final_vmaf, final_vmaf_time = _calculate_final_vmaf(
-        original_video_path, final_output_path, successful_scenes, 
-        calculate_full_video_vmaf, vmaf_config, config, logging_enabled
-    )
-    
-    # Generate comprehensive report
-    comprehensive_report = _generate_comprehensive_report(
-        original_video_path, final_output_path, successful_scenes, failed_scenes,
-        part5_start_time, merge_time, final_vmaf_time, config, logging_enabled
-    )
-    
-    # Save reports if enabled
-    if save_comprehensive_report:
-        _save_reports(comprehensive_report, output_dir, original_video_name, logging_enabled)
-    
-    if logging_enabled:
-        print(f"✅ Part 4 completed successfully!")
-        print(f"   📁 Final video: {os.path.basename(final_output_path)}")
-        print(f"   🎯 Final VMAF: {final_vmaf:.2f}" if final_vmaf else "   🎯 Final VMAF: Not calculated")
-    
-    return final_output_path, final_vmaf, comprehensive_report
-
-
-# ============================================================================
-# Core Processing Functions
-# ============================================================================
-
-def _create_output_directory(output_dir: str, logging_enabled: bool) -> bool:
-    """Create output directory with error handling."""
+    # ✅ ENHANCED: Create output directory with error handling
     try:
         os.makedirs(output_dir, exist_ok=True)
         if logging_enabled:
             print(f"   ✅ Output directory ready: {output_dir}")
-        return True
     except Exception as e:
         if logging_enabled:
             print(f"   ❌ Failed to create output directory: {e}")
-        return False
-
-
-def _validate_encoded_scenes(
-    encoded_scenes_data: List[Dict[str, Any]], 
-    logging_enabled: bool
-) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[str]]:
-    """Validate encoded scenes and categorize them."""
+        return None, None, None
+    
+    # Initialize processing start time
+    part5_start_time = time.time()
+    
+    # ===== SCENE VALIDATION =====
     if logging_enabled:
         print(f"   🔍 Validating encoded scenes...")
     
@@ -164,250 +64,233 @@ def _validate_encoded_scenes(
         if scene_data.get('encoding_success', False) and scene_data.get('encoded_path'):
             encoded_path = scene_data.get('encoded_path')
             
-            # Enhanced file validation with timeout protection
-            if _validate_scene_file(encoded_path, scene_num, logging_enabled):
-                successful_scenes.append(scene_data)
-                
-                # Only add original scene file (temp split files) to cleanup
-                original_scene_path = scene_data.get('path')
-                if _should_cleanup_scene_file(original_scene_path, encoded_path):
-                    temp_files_to_cleanup.append(original_scene_path)
-            else:
+            # ✅ ENHANCED: Better file validation with timeout protection
+            try:
+                if os.path.exists(encoded_path):
+                    file_size = os.path.getsize(encoded_path)
+                    if file_size > 0:
+                        successful_scenes.append(scene_data)
+                        # ❌ REMOVED: Don't add encoded files to general cleanup
+                        # temp_files_to_cleanup.append(encoded_path)  # REMOVED THIS LINE
+                        
+                        # Only add original scene file (temp split files) to cleanup
+                        original_scene_path = scene_data.get('path')
+                        if (original_scene_path and 
+                            os.path.exists(original_scene_path) and 
+                            original_scene_path != encoded_path and  # Don't delete if same file
+                            'temp_scenes' in original_scene_path):  # Only delete temp scene splits
+                            temp_files_to_cleanup.append(original_scene_path)
+                        
+                        if logging_enabled:
+                            print(f"         ✅ Scene {scene_num} valid ({file_size/1024/1024:.1f} MB)")
+                    else:
+                        if logging_enabled:
+                            print(f"         ❌ Scene {scene_num} file is empty")
+                        failed_scenes.append(scene_data)
+                else:
+                    if logging_enabled:
+                        print(f"         ❌ Scene {scene_num} file not found: {encoded_path}")
+                    failed_scenes.append(scene_data)
+            except Exception as e:
+                if logging_enabled:
+                    print(f"         ❌ Scene {scene_num} validation error: {e}")
                 failed_scenes.append(scene_data)
         else:
             if logging_enabled:
                 print(f"         ❌ Scene {scene_num} encoding failed")
             failed_scenes.append(scene_data)
     
+    # Count scenes with VMAF data (for reporting only)
+    scenes_with_vmaf = sum(1 for scene in successful_scenes if scene.get('actual_vmaf') is not None)
+    
     if logging_enabled:
-        print(f"   📊 Scene validation complete: {len(successful_scenes)} successful, {len(failed_scenes)} failed")
+        print(f"   📊 Scene validation results:")
+        print(f"      ✅ Successfully encoded scenes: {len(successful_scenes)}")
+        print(f"      📊 Scenes with VMAF data: {scenes_with_vmaf}")
+        print(f"      🔍 Scenes without VMAF: {len(successful_scenes) - scenes_with_vmaf}")
+        if failed_scenes:
+            print(f"      ❌ Failed scenes: {len(failed_scenes)}")
     
-    return successful_scenes, failed_scenes, temp_files_to_cleanup
-
-
-def _validate_scene_file(encoded_path: str, scene_num: int, logging_enabled: bool) -> bool:
-    """Validate a single scene file."""
-    try:
-        if os.path.exists(encoded_path):
-            file_size = os.path.getsize(encoded_path)
-            if file_size > 0:
-                if logging_enabled:
-                    print(f"         ✅ Scene {scene_num} valid ({file_size/1024/1024:.1f} MB)")
-                return True
-            else:
-                if logging_enabled:
-                    print(f"         ❌ Scene {scene_num} file is empty")
-                return False
-        else:
-            if logging_enabled:
-                print(f"         ❌ Scene {scene_num} file not found: {encoded_path}")
-            return False
-    except Exception as e:
+    if not successful_scenes:
         if logging_enabled:
-            print(f"         ❌ Scene {scene_num} validation error: {e}")
-        return False
-
-
-def _should_cleanup_scene_file(original_scene_path: Optional[str], encoded_path: str) -> bool:
-    """Determine if a scene file should be cleaned up."""
-    return (
-        original_scene_path and 
-        os.path.exists(original_scene_path) and 
-        original_scene_path != encoded_path and  # Don't delete if same file
-        'temp_scenes' in original_scene_path  # Only delete temp scene splits
-    )
-
-
-def _merge_video_scenes(
-    successful_scenes: List[Dict[str, Any]], 
-    output_dir: str, 
-    original_video_name: str, 
-    logging_enabled: bool
-) -> Tuple[Optional[str], float, float]:
-    """Merge video scenes into final output."""
+            print(f"   ❌ No successfully encoded scenes to process!")
+        return None, None, None
+    
+    # ===== REMOVED: CALCULATE MISSING VMAF SECTION =====
+    # We expect Part 4 to handle all VMAF calculations or skip them entirely
+    
     if logging_enabled:
-        print(f"   🔗 Merging {len(successful_scenes)} scenes into final video...")
+        print(f"   ✅ Using VMAF data from Part 4 (or skipped if disabled)")
     
-    # Generate output filename
+    # ===== VIDEO MERGING =====
+    if logging_enabled:
+        print(f"   🔗 Starting video merging process...")
+        print(f"      Merging {len(successful_scenes)} encoded scenes...")
+    
+    merge_start_time = time.time()
+    
+    # Generate timestamp for all uses (not just fallback)
     timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-    final_output_filename = f"{original_video_name}_compressed_{timestamp}.mp4"
+    # Use custom filename from config if available
+    custom_filename = config.get('output_settings', {}).get('custom_output_filename')
+
+    if custom_filename:
+        # Use the custom filename from Streamlit config
+        final_output_filename = custom_filename
+        if logging_enabled:
+            print(f"      📁 Using custom filename: {final_output_filename}")
+    else:
+        # Fallback to original pattern if no custom filename
+        codec_used = successful_scenes[0].get('codec_used', 'unknown') if successful_scenes else 'unknown'
+        final_output_filename = f"{original_video_name}_final_{codec_used}_{timestamp}.mp4"
+        if logging_enabled:
+            print(f"      📁 Using default pattern: {final_output_filename}")
+
     final_output_path = os.path.join(output_dir, final_output_filename)
     
     if logging_enabled:
-        print(f"   📁 Output: {final_output_filename}")
+        print(f"      📁 Output file: {final_output_filename}")
     
-    # Set up timeout handler for merging
-    def timeout_handler(signum, frame):
-        raise TimeoutError("Video merging timed out")
+    # Extract encoded scene paths in correct order
+    encoded_scene_paths = []
+    for scene_data in sorted(successful_scenes, key=lambda x: x.get('scene_number', 0)):
+        scene_path = scene_data['encoded_path']
+        encoded_scene_paths.append(scene_path)
+        if logging_enabled:
+            scene_size = os.path.getsize(scene_path) / 1024 / 1024 if os.path.exists(scene_path) else 0
+            print(f"         Scene {scene_data.get('scene_number')}: {os.path.basename(scene_path)} ({scene_size:.1f} MB)")
     
-    signal.signal(signal.SIGALRM, timeout_handler)
-    signal.alarm(MERGE_TIMEOUT_SECONDS)
+    # ✅ ENHANCED: Video merging with timeout and better error handling
+    if logging_enabled:
+        print(f"      🎬 Executing video merge...")
     
     try:
-        merge_start_time = time.time()
+        def timeout_handler(signum, frame):
+            raise TimeoutError("Video merging timeout")
+        
+        # Only set alarm on Unix systems
+        if hasattr(signal, 'alarm'):
+            signal.signal(signal.SIGALRM, timeout_handler)
+            signal.alarm(300)  # 5 minute timeout
         
-        # Perform video merging
-        merge_success = merge_videos(
-            scene_videos=[scene.get('encoded_path') for scene in successful_scenes],
-            output_video=final_output_path
-        )
+        try:
+            merge_success = merge_videos(encoded_scene_paths, final_output_path, logging_enabled=logging_enabled)
+        finally:
+            # Cancel the alarm
+            if hasattr(signal, 'alarm'):
+                signal.alarm(0)
         
         merge_time = time.time() - merge_start_time
-        signal.alarm(0)  # Cancel timeout
         
-        if not merge_success or not os.path.exists(final_output_path):
+        if not merge_success:
             if logging_enabled:
-                print(f"   ❌ Video merging failed")
-            return None, merge_time, 0.0
+                print(f"   ❌ Video merging failed!")
+            return None, None, None
         
-        # Get merged file size
-        merged_size = os.path.getsize(final_output_path) / (1024 * 1024)  # MB
+        # ✅ ENHANCED: Verify merged file
+        if not os.path.exists(final_output_path):
+            if logging_enabled:
+                print(f"   ❌ Merged file was not created!")
+            return None, None, None
+        
+        merged_size = os.path.getsize(final_output_path) / 1024 / 1024
+        if merged_size == 0:
+            if logging_enabled:
+                print(f"   ❌ Merged file is empty!")
+            return None, None, None
         
         if logging_enabled:
             print(f"   ✅ Video merging completed in {merge_time:.1f}s")
             print(f"   📁 Final video: {final_output_filename} ({merged_size:.1f} MB)")
-        
-        return final_output_path, merge_time, merged_size
-        
+        if cleanup_temp_files:
+            if logging_enabled:
+                print(f"   🧹 Cleaning up encoded scene files...")
+            
+            encoded_cleanup_success = 0
+            encoded_cleanup_failed = 0
+            
+            for scene_data in successful_scenes:
+                encoded_path = scene_data.get('encoded_path')
+                if encoded_path and os.path.exists(encoded_path):
+                    try:
+                        os.remove(encoded_path)
+                        encoded_cleanup_success += 1
+                        if logging_enabled:
+                            print(f"      🗑️ Deleted: {os.path.basename(encoded_path)}")
+                    except Exception as e:
+                        encoded_cleanup_failed += 1
+                        if logging_enabled:
+                            print(f"      ❌ Failed to delete {os.path.basename(encoded_path)}: {e}")
+            
+            if logging_enabled:
+                print(f"   ✅ Encoded scenes cleanup: {encoded_cleanup_success} deleted, {encoded_cleanup_failed} failed")
+    
     except TimeoutError:
-        signal.alarm(0)
         if logging_enabled:
-            print(f"   ❌ Video merging timed out after {MERGE_TIMEOUT_SECONDS} minutes!")
-        return None, 0.0, 0.0
+            print(f"   ❌ Video merging timed out after 5 minutes!")
+        return None, None, None
     except Exception as e:
-        signal.alarm(0)
         if logging_enabled:
             print(f"   ❌ Video merging failed with error: {e}")
             import traceback
             traceback.print_exc()
-        return None, 0.0, 0.0
-
-
-def _cleanup_temp_files(
-    temp_files_to_cleanup: List[str], 
-    successful_scenes: List[Dict[str, Any]], 
-    logging_enabled: bool
-) -> None:
-    """Clean up temporary files."""
-    if logging_enabled:
-        print(f"   🧹 Cleaning up temporary files...")
-    
-    # Clean up original scene files
-    temp_cleanup_success = 0
-    temp_cleanup_failed = 0
-    
-    for temp_file in temp_files_to_cleanup:
-        if os.path.exists(temp_file):
-            try:
-                os.remove(temp_file)
-                temp_cleanup_success += 1
-                if logging_enabled:
-                    print(f"      🗑️ Deleted: {os.path.basename(temp_file)}")
-            except Exception as e:
-                temp_cleanup_failed += 1
-                if logging_enabled:
-                    print(f"      ❌ Failed to delete {os.path.basename(temp_file)}: {e}")
-    
-    # Clean up encoded scene files
-    if logging_enabled:
-        print(f"   🧹 Cleaning up encoded scene files...")
-    
-    encoded_cleanup_success = 0
-    encoded_cleanup_failed = 0
-    
-    for scene_data in successful_scenes:
-        encoded_path = scene_data.get('encoded_path')
-        if encoded_path and os.path.exists(encoded_path):
-            try:
-                os.remove(encoded_path)
-                encoded_cleanup_success += 1
-                if logging_enabled:
-                    print(f"      🗑️ Deleted: {os.path.basename(encoded_path)}")
-            except Exception as e:
-                encoded_cleanup_failed += 1
-                if logging_enabled:
-                    print(f"      ❌ Failed to delete {os.path.basename(encoded_path)}: {e}")
+        return None, None, None
     
-    if logging_enabled:
-        print(f"   ✅ Cleanup complete: {temp_cleanup_success + encoded_cleanup_success} deleted, {temp_cleanup_failed + encoded_cleanup_failed} failed")
-
-
-def _calculate_final_vmaf(
-    original_video_path: str, 
-    final_output_path: str, 
-    successful_scenes: List[Dict[str, Any]], 
-    calculate_full_video_vmaf: bool, 
-    vmaf_config: Dict[str, Any], 
-    config: Dict[str, Any], 
-    logging_enabled: bool
-) -> Tuple[Optional[float], float]:
-    """Calculate final VMAF score."""
+    # ===== FINAL VMAF CALCULATION =====
     final_vmaf = None
     final_vmaf_time = 0
     
-    calculate_full_video_vmaf = False
-    
-    if calculate_full_video_vmaf:
-        if logging_enabled:
-            print(f"   📊 Starting final VMAF calculation...")
+    # calculate_full_video_vmaf = False
+
+    # if calculate_full_video_vmaf:
+    #     if logging_enabled:
+    #         print(f"   📊 Starting final VMAF calculation...")
         
-        try:
-            final_vmaf_start_time = time.time()
+    #     try:
+    #         final_vmaf_start_time = time.time()
             
-            final_vmaf = calculate_vmaf_advanced(
-                input_file=original_video_path,
-                encoded_file=final_output_path,
-                use_downscaling=vmaf_config.get('vmaf_use_downscaling', True),
-                scale_factor=vmaf_config.get('vmaf_downscaling_scale_factor', 0.5),
-                use_vmafneg=config.get('vmaf_models', {}).get('use_neg_by_default', False),
-                default_vmaf_model_path_config=config.get('vmaf_models', {}).get('default_model_path'),
-                vmafneg_model_path_config=config.get('vmaf_models', {}).get('neg_model_path'),
-                use_frame_rate_scaling=vmaf_config.get('vmaf_use_frame_rate_scaling', False),
-                target_fps=vmaf_config.get('vmaf_target_fps', 15.0),
-                frame_rate_scaling_method=vmaf_config.get('vmaf_frame_rate_scaling_method', 'uniform'),
-                logger=None,
-                logging_enabled=logging_enabled
-            )
-            final_vmaf_time = time.time() - final_vmaf_start_time
+    #         # Import VMAF calculation function
+    #         from calculate_vmaf_adv import calculate_vmaf_advanced
             
-            if logging_enabled:
-                print(f"   ✅ Final VMAF calculation completed in {final_vmaf_time:.1f}s")
-                print(f"   🎯 Final VMAF score: {final_vmaf:.2f}")
+    #         final_vmaf = calculate_vmaf_advanced(
+    #             input_file=original_video_path,
+    #             encoded_file=final_output_path,
+    #             use_downscaling=vmaf_config.get('vmaf_use_downscaling', True),
+    #             scale_factor=vmaf_config.get('vmaf_downscaling_scale_factor', 0.5),
+    #             use_vmafneg=config.get('vmaf_models', {}).get('use_neg_by_default', False),
+    #             default_vmaf_model_path_config=config.get('vmaf_models', {}).get('default_model_path'),
+    #             vmafneg_model_path_config=config.get('vmaf_models', {}).get('neg_model_path'),
+    #             use_frame_rate_scaling=vmaf_config.get('vmaf_use_frame_rate_scaling', False),
+    #             target_fps=vmaf_config.get('vmaf_target_fps', 15.0),
+    #             frame_rate_scaling_method=vmaf_config.get('vmaf_frame_rate_scaling_method', 'uniform'),
+    #             logger=None,
+    #             logging_enabled=logging_enabled
+    #         )
+    #         final_vmaf_time = time.time() - final_vmaf_start_time
+            
+    #         if logging_enabled:
+    #             print(f"   ✅ Final VMAF calculation completed in {final_vmaf_time:.1f}s")
+    #             print(f"   🎯 Final VMAF score: {final_vmaf:.2f}")
         
-        except Exception as e:
-            if logging_enabled:
-                print(f"   ❌ Final VMAF calculation failed: {e}")
-            final_vmaf = None
-    else:
-        if logging_enabled:
-            print(f"   ⏭️ Skipping full video VMAF calculation (disabled in config)")
+    #     except Exception as e:
+    #         if logging_enabled:
+    #             print(f"   ❌ Final VMAF calculation failed: {e}")
+    #         final_vmaf = None
+    # else:
+    if logging_enabled:
+        print(f"   ⏭️ Skipping full video VMAF calculation (disabled in config)")
     
     # Estimate final VMAF from scene averages if available
-    if not final_vmaf:
-        scene_vmafs = [scene.get('actual_vmaf', 0) for scene in successful_scenes if scene.get('actual_vmaf', 0) > 0]
-        if scene_vmafs:
-            final_vmaf = sum(scene_vmafs) / len(scene_vmafs)
-            if logging_enabled:
-                print(f"   📊 Estimated VMAF from scene average: {final_vmaf:.2f}")
-        else:
-            if logging_enabled:
-                print(f"   ⏭️ No scene VMAF data available for estimation")
+    scene_vmafs = [scene.get('actual_vmaf', 0) for scene in successful_scenes if scene.get('actual_vmaf', 0) > 0]
+    if scene_vmafs:
+        final_vmaf = sum(scene_vmafs) / len(scene_vmafs)
+        if logging_enabled:
+            print(f"   📊 Estimated VMAF from scene average: {final_vmaf:.2f}")
+    else:
+        if logging_enabled:
+            print(f"   ⏭️ No scene VMAF data available for estimation")
     
-    return final_vmaf, final_vmaf_time
-
-
-def _generate_comprehensive_report(
-    original_video_path: str, 
-    final_output_path: str, 
-    successful_scenes: List[Dict[str, Any]], 
-    failed_scenes: List[Dict[str, Any]], 
-    part5_start_time: float, 
-    merge_time: float, 
-    final_vmaf_time: float, 
-    config: Dict[str, Any], 
-    logging_enabled: bool
-) -> Dict[str, Any]:
-    """Generate comprehensive processing report."""
+    # ===== COMPREHENSIVE REPORT GENERATION =====
     if logging_enabled:
         print(f"   📋 Generating comprehensive report...")
 
@@ -424,207 +307,293 @@ def _generate_comprehensive_report(
     target_vmaf = config.get('video_processing', {}).get('target_vmaf', 93.0)
     scenes_meeting_target = sum(1 for s in scenes_with_valid_vmaf if s['actual_vmaf'] >= target_vmaf)
 
-    # Extract training data from scenes
-    training_data = _extract_training_data_from_scenes(successful_scenes)
+    # ✅ SIMPLE: Extract training data directly from scenes
+    def extract_training_data_from_scenes(scenes_data):
+        """Extract comprehensive training data directly from scene data."""
+        training_data = {
+            'pipeline_stages_data': {
+                'part1_preprocessing': [],
+                'part2_scene_detection': [],
+                'part3_ai_encoding': [],
+                'part4_vmaf_calculation': [],
+                'part5_merging': []
+            },
+            'model_training_features': [],
+            'prediction_accuracy_analysis': [],
+            'scene_classification_results': [],
+            'encoding_optimization_data': [],
+            'vmaf_validation_data': [],
+            'feature_extraction_performance': []
+        }
+        
+        for scene in scenes_data:
+            scene_num = scene.get('scene_number', 'unknown')
+            
+            # Extract Part 1 data (preprocessing info)
+            if 'original_video_metadata' in scene:
+                part1_data = scene['original_video_metadata'].copy()
+                part1_data['scene_number'] = scene_num
+                training_data['pipeline_stages_data']['part1_preprocessing'].append(part1_data)
+            
+            # Extract Part 2 data (scene detection)
+            part2_data = {
+                'scene_number': scene_num,
+                'start_time': scene.get('start_time'),
+                'end_time': scene.get('end_time'),
+                'duration': scene.get('duration'),
+                'scene_codec': scene.get('scene_codec'),
+                'scene_container': scene.get('scene_container'),
+                'file_size_mb': scene.get('file_size_mb'),
+                'scene_detection_method': 'content_based' if len(scenes_data) > 1 else 'time_based'
+            }
+            training_data['pipeline_stages_data']['part2_scene_detection'].append(part2_data)
+            
+            # Extract Part 3 data (AI encoding) - this is the richest data source
+            if scene.get('encoding_success', False):
+                part3_data = {
+                    'scene_number': scene_num,
+                    'scene_type': scene.get('scene_type'),
+                    'codec_used': scene.get('codec_used'),
+                    'final_adjusted_cq': scene.get('final_adjusted_cq'),
+                    'predicted_vmaf_at_optimal_cq': scene.get('predicted_vmaf_at_optimal_cq'),
+                    'processing_time_seconds': scene.get('processing_time_seconds'),
+                    'compression_ratio': scene.get('compression_ratio'),
+                    'size_ratio': scene.get('size_ratio'),
+                    'encoding_success': True,
+                    'input_size_mb': scene.get('input_size_mb'),
+                    'encoded_file_size_mb': scene.get('encoded_file_size_mb')
+                }
+                training_data['pipeline_stages_data']['part3_ai_encoding'].append(part3_data)
+                
+                # Extract model training data if available
+                model_data = scene.get('model_training_data', {})
+                if model_data:
+                    model_features = {
+                        'scene_number': scene_num,
+                        'raw_video_features': model_data.get('raw_video_features', {}),
+                        'processed_video_features': model_data.get('processed_video_features', {}),
+                        'vmaf_model_features': model_data.get('vmaf_model_features', {}),
+                        'scene_classifier_features': model_data.get('scene_classifier_features', {}),
+                        'scene_classifier_probabilities': model_data.get('scene_classifier_probabilities', {}),
+                        'processing_timings': model_data.get('processing_timings', {})
+                    }
+                    training_data['model_training_features'].append(model_features)
+                    
+                    # Extract prediction accuracy
+                    prediction_acc = model_data.get('prediction_accuracy', {})
+                    if prediction_acc:
+                        training_data['prediction_accuracy_analysis'].append({
+                            'scene_number': scene_num,
+                            'predicted_vmaf': prediction_acc.get('predicted_vmaf'),
+                            'prediction_error': prediction_acc.get('prediction_error'),
+                            'target_vmaf': scene.get('target_vmaf'),
+                            'actual_vmaf': scene.get('actual_vmaf')
+                        })
+                    
+                    # Extract feature extraction performance
+                    if model_data.get('feature_recovery_attempts'):
+                        training_data['feature_extraction_performance'].append({
+                            'scene_number': scene_num,
+                            'recovery_attempts': model_data.get('feature_recovery_attempts', []),
+                            'feature_extraction_successful': bool(model_data.get('raw_video_features'))
+                        })
+                
+                # Scene classification results
+                if scene.get('scene_type'):
+                    training_data['scene_classification_results'].append({
+                        'scene_number': scene_num,
+                        'classified_type': scene.get('scene_type'),
+                        'classification_confidence': scene.get('scene_classification_confidence'),
+                        'duration': scene.get('duration'),
+                        'file_size_mb': scene.get('file_size_mb')
+                    })
+                
+                # Encoding optimization data
+                training_data['encoding_optimization_data'].append({
+                    'scene_number': scene_num,
+                    'scene_type': scene.get('scene_type'),
+                    'optimal_cq': scene.get('optimal_cq'),
+                    'final_adjusted_cq': scene.get('final_adjusted_cq'),
+                    'conservative_adjustment': scene.get('conservative_cq_adjustment', 0),
+                    'size_protection_used': scene.get('size_protection_used', False),
+                    'compression_achieved': scene.get('compression_ratio', 0),
+                    'target_vmaf': scene.get('target_vmaf'),
+                    'predicted_vmaf': scene.get('predicted_vmaf_at_optimal_cq')
+                })
+            
+            # Extract Part 4 data (VMAF calculation)
+            part4_data = {
+                'scene_number': scene_num,
+                'actual_vmaf': scene.get('actual_vmaf'),
+                'vmaf_calculation_status': scene.get('vmaf_calculation_status'),
+                'vmaf_calculation_time': scene.get('vmaf_calculation_time', 0),
+                'vmaf_calculation_notes': scene.get('vmaf_calculation_notes'),
+                'target_achieved': scene.get('actual_vmaf', 0) >= scene.get('target_vmaf', 0) if scene.get('actual_vmaf') else None
+            }
+            training_data['pipeline_stages_data']['part4_vmaf_calculation'].append(part4_data)
+            
+            # VMAF validation data (detailed)
+            if scene.get('actual_vmaf') is not None:
+                vmaf_data = {
+                    'scene_number': scene_num,
+                    'actual_vmaf': scene.get('actual_vmaf'),
+                    'predicted_vmaf': scene.get('predicted_vmaf_at_optimal_cq'),
+                    'target_vmaf': scene.get('target_vmaf'),
+                    'vmaf_config_used': scene.get('vmaf_config_used', {}),
+                    'calculation_method': scene.get('vmaf_calculation_notes', '').split(' ')[0] if scene.get('vmaf_calculation_notes') else 'unknown'
+                }
+                
+                # Add prediction analysis if available
+                if scene.get('vmaf_prediction_analysis'):
+                    vmaf_data.update(scene['vmaf_prediction_analysis'])
+                
+                training_data['vmaf_validation_data'].append(vmaf_data)
+        
+        # Add Part 4 data (merging performance)
+        training_data['pipeline_stages_data']['part5_merging'] = {
+            'total_scenes_processed': len(scenes_data),
+            'successful_scenes': len([s for s in scenes_data if s.get('encoding_success')]),
+            'merging_time': merge_time,
+            'final_vmaf_calculation_time': final_vmaf_time,
+            'total_part5_time': total_processing_time,
+            'overall_compression_ratio': overall_compression_ratio
+        }
+        
+        return training_data
+
+    # Extract training data
+    comprehensive_training_data = extract_training_data_from_scenes(successful_scenes)
 
-    # Create comprehensive report
     comprehensive_report = {
         'processing_summary': {
+            'timestamp': datetime.now().isoformat(),
             'original_video_path': original_video_path,
             'final_video_path': final_output_path,
-            'total_processing_time_seconds': total_processing_time,
-            'merge_time_seconds': merge_time,
-            'final_vmaf_calculation_time_seconds': final_vmaf_time,
-            'timestamp': datetime.now().isoformat()
-        },
-        'compression_metrics': {
-            'input_size_total_mb': input_size_total,
-            'output_size_total_mb': output_size_total,
-            'overall_compression_ratio_percent': overall_compression_ratio,
-            'scenes_processed': len(successful_scenes),
-            'scenes_failed': len(failed_scenes)
+            'total_scenes_processed': len(successful_scenes),
+            'failed_scenes_count': len(failed_scenes),
+            'processing_time_seconds': total_processing_time,
+            'vmaf_calculation_enabled': config.get('vmaf_calculation', {}).get('calculate_scene_vmaf', True)
         },
+        
         'quality_metrics': {
-            'final_vmaf': None,  # Will be updated later
+            'final_vmaf_score': final_vmaf,
+            'final_vmaf_calculated': calculate_full_video_vmaf,
+            'final_vmaf_source': 'calculated' if calculate_full_video_vmaf else 'estimated_from_scenes' if scenes_with_valid_vmaf else 'unavailable',
             'average_scene_vmaf': average_scene_vmaf,
+            'target_vmaf': target_vmaf,
+            'target_achieved': (final_vmaf >= target_vmaf) if final_vmaf else False,
             'scenes_meeting_target': scenes_meeting_target,
-            'total_scenes_with_vmaf': len(scenes_with_valid_vmaf),
-            'target_vmaf': target_vmaf
+            'scenes_with_vmaf_data': len(scenes_with_valid_vmaf),
+            'vmaf_calculation_time': final_vmaf_time,
+            'scene_vmaf_source': 'part4'
+        },
+        
+        'compression_metrics': {
+            'total_input_size_mb': input_size_total,
+            'total_output_size_mb': output_size_total,
+            'size_ratio': output_size_total / input_size_total if input_size_total > 0 else 0,
+            'overall_compression_ratio_percent': overall_compression_ratio,
+            'final_file_size_mb': os.path.getsize(final_output_path) / (1024 * 1024) if os.path.exists(final_output_path) else 0
         },
+        
         'scene_analysis': {
-            'successful_scenes': successful_scenes,
-            'failed_scenes': failed_scenes,
-            'training_data': training_data
-        }
-    }
-
-    return comprehensive_report
-
-
-def _extract_training_data_from_scenes(scenes_data: List[Dict[str, Any]]) -> Dict[str, Any]:
-    """Extract comprehensive training data directly from scene data."""
-    training_data = {
-        'pipeline_stages_data': {
-            'part1_preprocessing': [],
-            'part2_scene_detection': [],
-            'part3_ai_encoding': [],
-            'part4_vmaf_calculation': [],
-            'part5_merging': []
+            'scene_types_distribution': {},
+            'codec_usage': {},
+            'prediction_accuracy_stats': {
+                'scenes_meeting_target': scenes_meeting_target,
+                'scenes_with_vmaf': len(scenes_with_valid_vmaf),
+                'average_scene_vmaf': average_scene_vmaf
+            }
         },
-        'model_training_features': [],
-        'prediction_accuracy_analysis': [],
-        'scene_classification_results': [],
-        'encoding_optimization_data': [],
-        'vmaf_validation_data': [],
-        'feature_extraction_performance': []
-    }
-    
-    for scene in scenes_data:
-        scene_num = scene.get('scene_number', 'unknown')
         
-        # Extract Part 1 data (preprocessing info)
-        if 'original_video_metadata' in scene:
-            part1_data = scene['original_video_metadata'].copy()
-            part1_data['scene_number'] = scene_num
-            training_data['pipeline_stages_data']['part1_preprocessing'].append(part1_data)
+        'processing_performance': {
+            'part5_merging_time': merge_time,
+            'part5_final_vmaf_time': final_vmaf_time,
+            'part5_total_time': total_processing_time
+        },
         
-        # Extract Part 2 data (scene detection)
-        part2_data = {
-            'scene_number': scene_num,
-            'start_time': scene.get('start_time'),
-            'end_time': scene.get('end_time'),
-            'duration': scene.get('duration'),
-            'scene_codec': scene.get('scene_codec'),
-            'scene_container': scene.get('scene_container'),
-            'file_size_mb': scene.get('file_size_mb'),
-            'scene_detection_method': 'content_based' if len(scenes_data) > 1 else 'time_based'
-        }
-        training_data['pipeline_stages_data']['part2_scene_detection'].append(part2_data)
+        'individual_scenes_summary': [],
         
-        # Extract Part 3 data (AI encoding) - this is the richest data source
-        if scene.get('encoding_success', False):
-            part3_data = {
-                'scene_number': scene_num,
-                'scene_type': scene.get('scene_type'),
-                'codec_used': scene.get('codec_used'),
-                'final_adjusted_cq': scene.get('final_adjusted_cq'),
-                'predicted_vmaf_at_optimal_cq': scene.get('predicted_vmaf_at_optimal_cq'),
-                'processing_time_seconds': scene.get('processing_time_seconds'),
-                'compression_ratio': scene.get('compression_ratio'),
-                'size_ratio': scene.get('size_ratio'),
-                'encoding_success': True,
-                'input_size_mb': scene.get('input_size_mb'),
-                'encoded_file_size_mb': scene.get('encoded_file_size_mb')
-            }
-            training_data['pipeline_stages_data']['part3_ai_encoding'].append(part3_data)
-            
-            # Extract model training data if available
-            model_data = scene.get('model_training_data', {})
-            if model_data:
-                _extract_model_training_data(training_data, scene_num, model_data, scene)
-    
-    return training_data
-
-
-def _extract_model_training_data(
-    training_data: Dict[str, Any], 
-    scene_num: Any, 
-    model_data: Dict[str, Any], 
-    scene: Dict[str, Any]
-) -> None:
-    """Extract model training data from scene."""
-    # Extract model features
-    model_features = {
-        'scene_number': scene_num,
-        'raw_video_features': model_data.get('raw_video_features', {}),
-        'processed_video_features': model_data.get('processed_video_features', {}),
-        'vmaf_model_features': model_data.get('vmaf_model_features', {}),
-        'scene_classifier_features': model_data.get('scene_classifier_features', {}),
-        'scene_classifier_probabilities': model_data.get('scene_classifier_probabilities', {}),
-        'processing_timings': model_data.get('processing_timings', {})
+        # ✅ ENHANCED: Add comprehensive training data directly extracted from scenes
+        'comprehensive_training_data': comprehensive_training_data
     }
-    training_data['model_training_features'].append(model_features)
-    
-    # Extract prediction accuracy
-    prediction_acc = model_data.get('prediction_accuracy', {})
-    if prediction_acc:
-        training_data['prediction_accuracy_analysis'].append({
-            'scene_number': scene_num,
-            'predicted_vmaf': prediction_acc.get('predicted_vmaf'),
-            'prediction_error': prediction_acc.get('prediction_error'),
-            'target_vmaf': scene.get('target_vmaf'),
-            'actual_vmaf': scene.get('actual_vmaf')
-        })
-    
-    # Extract feature extraction performance
-    if model_data.get('feature_recovery_attempts'):
-        training_data['feature_extraction_performance'].append({
-            'scene_number': scene_num,
-            'recovery_attempts': model_data.get('feature_recovery_attempts', []),
-            'feature_extraction_successful': bool(model_data.get('raw_video_features'))
-        })
-
-
-def _save_reports(
-    comprehensive_report: Dict[str, Any], 
-    output_dir: str, 
-    original_video_name: str, 
-    logging_enabled: bool
-) -> None:
-    """Save comprehensive and individual reports."""
-    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-    
-    # Save comprehensive report
-    comprehensive_report_path = os.path.join(
-        output_dir, 
-        f"comprehensive_processing_report_{original_video_name}_{timestamp}.json"
-    )
-    
-    try:
-        with open(comprehensive_report_path, 'w') as f:
-            json.dump(comprehensive_report, f, indent=2, default=str)
         
-        if logging_enabled:
-            print(f"   📊 Comprehensive report saved: {os.path.basename(comprehensive_report_path)}")
-    except Exception as e:
-        if logging_enabled:
-            print(f"   ❌ Failed to save comprehensive report: {e}")
     
-    # Save individual scene reports if enabled
-    if comprehensive_report.get('scene_analysis', {}).get('successful_scenes'):
-        _save_individual_scene_reports(
-            comprehensive_report['scene_analysis']['successful_scenes'],
-            output_dir, original_video_name, timestamp, logging_enabled
-        )
-
+    # ===== SAVE REPORTS =====
+    if save_comprehensive_report:
+        if logging_enabled:
+            print(f"   💾 Saving comprehensive processing report...")
 
-def _save_individual_scene_reports(
-    successful_scenes: List[Dict[str, Any]], 
-    output_dir: str, 
-    original_video_name: str, 
-    timestamp: str, 
-    logging_enabled: bool
-) -> None:
-    """Save individual scene reports."""
-    scene_reports_dir = os.path.join(output_dir, 'scene_reports')
-    os.makedirs(scene_reports_dir, exist_ok=True)
-    
-    for scene in successful_scenes:
-        scene_num = scene.get('scene_number', 'unknown')
-        scene_report_path = os.path.join(
-            scene_reports_dir, 
-            f"scene_{scene_num}_report_{original_video_name}_{timestamp}.json"
-        )
-        
         try:
-            with open(scene_report_path, 'w') as f:
-                json.dump(scene, f, indent=2, default=str)
+            # ✅ SIMPLE: Extract original filename from the original_video_path parameter
+            original_name_from_path = os.path.splitext(os.path.basename(original_video_path))[0]
+            
+            # Clean up temp prefixes if present
+            if original_name_from_path.startswith('tmp'):
+                # Try to get from config if available
+                config_original_name = config.get('output_settings', {}).get('original_video_name')
+                if config_original_name:
+                    clean_name = config_original_name
+                else:
+                    # Try to extract from custom filename in config
+                    custom_filename = config.get('output_settings', {}).get('custom_output_filename')
+                    if custom_filename and '_encoded_' in custom_filename:
+                        clean_name = custom_filename.split('_encoded_')[0]
+                    else:
+                        clean_name = "video"  # Generic fallback
+            else:
+                clean_name = original_name_from_path
+            
+            report_filename = f"{clean_name}_encoding_report.json"
+            report_path = os.path.join(output_dir, report_filename)
+            
+            if logging_enabled:
+                print(f"      📁 Report filename: {report_filename}")
+                print(f"      📊 Extracted from: {os.path.basename(original_video_path)}")
+            
+            with open(report_path, 'w') as f:
+                json.dump(comprehensive_report, f, indent=2, default=str)
+            
+            if logging_enabled:
+                print(f"      📄 Saved comprehensive report: {report_filename}")
         except Exception as e:
             if logging_enabled:
-                print(f"   ❌ Failed to save scene {scene_num} report: {e}")
+                print(f"      ❌ Failed to save comprehensive report: {e}")
+    # ===== CLEANUP TEMPORARY FILES =====
+    if cleanup_temp_files and temp_files_to_cleanup:
+        if logging_enabled:
+            print(f"   🧹 Cleaning up temporary files...")
+        
+        cleanup_success = 0
+        cleanup_failed = 0
+        
+        for temp_file in set(temp_files_to_cleanup):
+            try:
+                if os.path.exists(temp_file):
+                    os.remove(temp_file)
+                    cleanup_success += 1
+                    if logging_enabled:
+                        print(f"      🗑️ Deleted: {os.path.basename(temp_file)}")
+            except Exception as e:
+                cleanup_failed += 1
+                if logging_enabled:
+                    print(f"      ❌ Failed to delete {os.path.basename(temp_file)}: {e}")
+        
+        if logging_enabled:
+            print(f"   ✅ Cleanup completed: {cleanup_success} files deleted, {cleanup_failed} failed")
     
+    # ===== FINAL SUMMARY =====
     if logging_enabled:
-        print(f"   📄 Individual scene reports saved in: scene_reports/")
+        print(f"\n🎉 --- Part 4 Completed Successfully ---")
+        print(f"   📁 Final video: {final_output_filename}")
+        if final_vmaf:
+            print(f"   🎯 Final VMAF: {final_vmaf:.2f} (target: {target_vmaf:.1f}) - {'✅ ACHIEVED' if final_vmaf >= target_vmaf else '❌ MISSED'}")
+        print(f"   🗜️ Overall compression: {overall_compression_ratio:+.1f}%")
+        print(f"   ⏱️ Total processing time: {total_processing_time:.1f}s")
+        print(f"   📊 Scenes meeting target: {scenes_meeting_target}/{len(scenes_with_valid_vmaf)}")
+        print(f"   🔍 VMAF data from Part 4: {len(scenes_with_valid_vmaf)}/{len(successful_scenes)} scenes")
+    
+    return final_output_path, final_vmaf, comprehensive_report
 
 
 

services/compress/video_preprocessor.py

@@ -1,53 +1,8 @@
-"""
-Video Pre-processing Module
-
-This module handles the initial video processing stage including:
-- Video validation and duration checks
-- Codec detection and handling
-- Lossless format conversion when necessary
-"""
-
 import os
-from datetime import datetime
-from pathlib import Path
-from typing import Optional, Dict, Any
-
 from utils.video_utils import get_video_duration, get_video_codec
 from utils.encode_video import encode_lossless_video
 
-
-# ============================================================================
-# Constants
-# ============================================================================
-
-# Quality to VMAF score mapping
-QUALITY_VMAF_MAPPING = {
-    'High': 95.0,
-    'Medium': 90.0,  # Updated from 93.0 to 90.0
-    'Low': 85.0      # Updated from 90.0 to 85.0
-}
-
-# Lossless codecs that require re-encoding
-LOSSLESS_CODECS = [
-    'ffv1', 'h264_lossless', 'utvideo', 'rawvideo', 
-    'prores_ks', 'dnxhd', 'cineform'
-]
-
-# Lossless file extensions
-LOSSLESS_EXTENSIONS = ['.y4m', '.yuv', '.raw']
-
-
-# ============================================================================
-# Main Function
-# ============================================================================
-
-def pre_processing(
-    video_path: str, 
-    target_quality: str = 'Medium',
-    codec: str = 'auto', 
-    max_duration: int = 60, 
-    output_dir: str = './output'
-) -> Optional[Dict[str, Any]]:
+def pre_processing(video_path, target_quality='Medium',codec='auto', max_duration=60, output_dir='./output'):
     """
     Part 1: Initial video checks and lossless encoding if necessary.
 
@@ -59,88 +14,52 @@ def pre_processing(
     unmodified, while videos exceeding the maximum duration are rejected.
 
     Args:
-        video_path: The full path to the input video file
-        target_quality: Target quality level - 'High', 'Medium', or 'Low'
-                       Gets converted to VMAF scores: High=95, Medium=93, Low=90
-        max_duration: Maximum allowed video duration in seconds. Default: 3600 (1 hour)
-        output_dir: Directory for final output files. Default: './output'
-        codec: Target encoding codec. Default: 'auto' (auto-detect best available)
-               Options: 'auto', 'av1_nvenc', 'libx264', 'libx265', 'h264_nvenc', etc.
+        video_path (str): The full path to the input video file.
+        target_quality (str): Target quality level - 'High', 'Medium', or 'Low'.
+                             Gets converted to VMAF scores: High=95, Medium=93, Low=90.
+        max_duration (int): Maximum allowed video duration in seconds. Default: 3600 (1 hour).
+        output_dir (str): Directory for final output files. Default: './output'.
+        codec (str): Target encoding codec. Default: 'auto' (auto-detect best available).
+                    Options: 'auto', 'av1_nvenc', 'libx264', 'libx265', 'h264_nvenc', etc.
+       
 
     Returns:
-        dict or None: Dictionary containing video metadata and processing info if successful, 
-                     otherwise None. Keys: 'path', 'codec', 'original_codec', 'duration', 
-                     'was_reencoded', 'encoding_time', 'target_vmaf', 'target_quality', 
-                     'directories'
+        dict or None: Dictionary containing video metadata and processing info if successful, otherwise None.
+                     Keys: 'path', 'codec', 'original_codec', 'duration', 'was_reencoded', 
+                           'encoding_time', 'target_vmaf', 'target_quality', 'directories'
     """
-    # Resolve codec selection
-    target_codec = _resolve_target_codec(codec)
+    
+    # ✅ CODEC HANDLING: Resolve auto codec selection
+    if codec.lower() == 'auto':
+        target_codec = 'av1_nvenc'  # Default to AV1 for best quality
+        
+    else:
+        target_codec = codec
+        print(f"🎯 Using specified codec: {target_codec}")
+    
+    # ✅ QUALITY MAPPING: Convert target quality to VMAF score
+    quality_vmaf_mapping = {
+        'High': 95.0,
+        'Medium': 90.0,  # Updated from 93.0 to 90.0
+        'Low': 85.0      # Updated from 90.0 to 85.0
+    }
     
     # Validate and convert target quality
-    target_quality, target_vmaf = _validate_target_quality(target_quality)
+    if target_quality not in quality_vmaf_mapping:
+        print(f"⚠️ Invalid target quality '{target_quality}'. Using 'Medium' as fallback.")
+        target_quality = 'Medium'
+    
+    target_vmaf = quality_vmaf_mapping[target_quality]
     
     print(f"🎯 Target quality: {target_quality} (VMAF: {target_vmaf})")
     print(f"🎥 Target codec: {target_codec}")
     
-    # Setup directories
-    _setup_output_directory(output_dir)
-    print(f"📁 Output directory: {output_dir}")
-    
-    # Validate video duration
-    duration = _validate_video_duration(video_path, max_duration)
-    if duration is None:
-        return None
-    
-    # Detect video codec
-    original_codec = _detect_video_codec(video_path)
-    if not original_codec:
-        return None
-    
-    # Check if video needs lossless conversion
-    is_lossless = _is_lossless_format(original_codec, video_path)
+    # ✅ DIRECTORY SETUP: Create output and temp directories
+    os.makedirs(output_dir, exist_ok=True)
     
-    if is_lossless:
-        return _handle_lossless_conversion(
-            video_path, original_codec, duration, target_vmaf, 
-            target_quality, target_codec, output_dir
-        )
-    else:
-        return _handle_compressed_video(
-            video_path, original_codec, duration, target_vmaf, 
-            target_quality, target_codec
-        )
-
-
-# ============================================================================
-# Helper Functions
-# ============================================================================
-
-def _resolve_target_codec(codec: str) -> str:
-    """Resolve auto codec selection to specific codec."""
-    if codec.lower() == 'auto':
-        return 'av1_nvenc'  # Default to AV1 for best quality
-    else:
-        print(f"🎯 Using specified codec: {codec}")
-        return codec
-
-
-def _validate_target_quality(target_quality: str) -> tuple[str, float]:
-    """Validate target quality and convert to VMAF score."""
-    if target_quality not in QUALITY_VMAF_MAPPING:
-        print(f"⚠️ Invalid target quality '{target_quality}'. Using 'Medium' as fallback.")
-        target_quality = 'Medium'
+    print(f"📁 Output directory: {output_dir}")
     
-    target_vmaf = QUALITY_VMAF_MAPPING[target_quality]
-    return target_quality, target_vmaf
-
-
-def _setup_output_directory(output_dir: str) -> None:
-    """Create output directory if it doesn't exist."""
-    os.makedirs(output_dir, exist_ok=True)
-
-
-def _validate_video_duration(video_path: str, max_duration: int) -> Optional[float]:
-    """Validate video duration against maximum allowed duration."""
+    # ✅ DURATION CHECK: Validate video duration
     print(f"⏱️ Checking video duration...")
     duration = get_video_duration(video_path)
     
@@ -153,11 +72,8 @@ def _validate_video_duration(video_path: str, max_duration: int) -> Optional[flo
         return None
     
     print(f"✅ Video duration: {duration}s (within limit: {max_duration}s)")
-    return duration
-
-
-def _detect_video_codec(video_path: str) -> Optional[str]:
-    """Detect video codec from file."""
+    
+    # CODEC CHECK: Determine video codec
     print(f"🎥 Checking video codec...")
     original_codec = get_video_codec(video_path)
     
@@ -166,99 +82,81 @@ def _detect_video_codec(video_path: str) -> Optional[str]:
         return None
     
     print(f"🎥 Detected codec: {original_codec}")
-    return original_codec
-
-
-def _is_lossless_format(codec: str, video_path: str) -> bool:
-    """Check if video is in a lossless format."""
-    return (
-        codec.lower() in LOSSLESS_CODECS or 
-        any(video_path.lower().endswith(ext) for ext in LOSSLESS_EXTENSIONS)
-    )
-
-
-def _handle_lossless_conversion(
-    video_path: str, 
-    original_codec: str, 
-    duration: float, 
-    target_vmaf: float, 
-    target_quality: str, 
-    target_codec: str, 
-    output_dir: str
-) -> Optional[Dict[str, Any]]:
-    """Handle conversion of lossless video to standardized format."""
-    print(f"🔄 Video is in a lossless format ({original_codec}). Re-encoding with standardized lossless compression...")
-    
-    # Generate output filename with timestamp
-    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-    base_name = os.path.splitext(os.path.basename(video_path))[0]
-    output_filename = f"{base_name}_lossless_{timestamp}.mkv"
-    output_path = os.path.join(output_dir, output_filename)
     
-    print(f"📁 Lossless output: {output_filename}")
+    # LOSSLESS CODEC HANDLING: Define lossless codecs and special cases
+    lossless_codecs = ['ffv1', 'h264_lossless', 'utvideo', 'rawvideo', 'prores_ks', 'dnxhd', 'cineform']
+    lossless_extensions = ['.y4m', '.yuv', '.raw']
     
-    # Perform lossless encoding
-    encode_log, encode_time = encode_lossless_video(
-        input_path=video_path,
-        output_path=output_path,
-        logging_enabled=True
+    is_lossless = (
+        original_codec.lower() in lossless_codecs or 
+        any(video_path.lower().endswith(ext) for ext in lossless_extensions)
     )
     
-    if encode_log:
-        print(f"✅ Lossless encoding successful in {encode_time:.1f}s")
-        print(f"📁 Output at: {output_path}")
+    if is_lossless:
+        print(f"🔄 Video is in a lossless format ({original_codec}). Re-encoding with standardized lossless compression...")
+        
+        # Generate output filename with timestamp to avoid conflicts
+        from datetime import datetime
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        base_name = os.path.splitext(os.path.basename(video_path))[0]
+        output_filename = f"{base_name}_lossless_{timestamp}.mkv"
+        output_path = os.path.join(output_dir, output_filename)
+        
+        print(f"📁 Lossless output: {output_filename}")
         
-        # Return comprehensive metadata
+        # ✅ IMPROVED: Use direct FFmpeg command for lossless encoding
+        # This bypasses the main encode_video function which expects lossy codecs
+        encode_log, encode_time = encode_lossless_video(
+            input_path=video_path,
+            output_path=output_path,
+            logging_enabled=True
+        )
+        
+        if encode_log:
+            print(f"✅ Lossless encoding successful in {encode_time:.1f}s")
+            print(f"📁 Output at: {output_path}")
+            
+            # Return comprehensive metadata
+            return {
+                'path': output_path,
+                'codec': 'ffv1',
+                'original_codec': original_codec,
+                'duration': duration,
+                'was_reencoded': True,
+                'encoding_time': encode_time,
+                'target_vmaf': target_vmaf,
+                'target_quality': target_quality,
+                'target_codec': target_codec,
+                'processing_info': {
+                    'lossless_conversion': True,
+                    'original_format': original_codec,
+                    'standardized_format': 'ffv1',
+                    'container': 'mkv'
+                }
+            }
+        else:
+            print("❌ Lossless encoding failed.")
+            return None
+    else:
+        print(f"✅ Video is already compressed with codec: {original_codec}. Proceeding with original file.")
+        
+        # Return metadata for compressed video
         return {
-            'path': output_path,
-            'codec': 'ffv1',
+            'path': video_path,
+            'codec': original_codec,
             'original_codec': original_codec,
             'duration': duration,
-            'was_reencoded': True,
-            'encoding_time': encode_time,
+            'was_reencoded': False,
+            'encoding_time': 0,
             'target_vmaf': target_vmaf,
             'target_quality': target_quality,
-            'target_codec': target_codec,
+            'target_codec': target_codec,  # ✅ Added target codec
             'processing_info': {
-                'lossless_conversion': True,
+                'lossless_conversion': False,
                 'original_format': original_codec,
-                'standardized_format': 'ffv1',
-                'container': 'mkv'
+                'standardized_format': original_codec,
+                'container': os.path.splitext(video_path)[1][1:]  # Extension without dot
             }
         }
-    else:
-        print("❌ Lossless encoding failed.")
-        return None
-
-
-def _handle_compressed_video(
-    video_path: str, 
-    original_codec: str, 
-    duration: float, 
-    target_vmaf: float, 
-    target_quality: str, 
-    target_codec: str
-) -> Dict[str, Any]:
-    """Handle compressed video that doesn't need conversion."""
-    print(f"✅ Video is already compressed with codec: {original_codec}. Proceeding with original file.")
-    
-    # Return metadata for compressed video
-    return {
-        'path': video_path,
-        'codec': original_codec,
-        'original_codec': original_codec,
-        'duration': duration,
-        'was_reencoded': False,
-        'encoding_time': 0,
-        'target_vmaf': target_vmaf,
-        'target_quality': target_quality,
-        'target_codec': target_codec,
-        'processing_info': {
-            'lossless_conversion': False,
-            'original_format': original_codec,
-            'standardized_format': original_codec,
-            'container': os.path.splitext(video_path)[1][1:]  # Extension without dot
-        }
-    }