update configs

services/compress/encoder.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import torch
 import time
 import subprocess
@@ -140,7 +141,7 @@ def get_cq_from_lookup_table(scene_type, config, target_vmaf=None, target_qualit
 #         "class_mapping": class_mapping
 #     }
 
-def get_scalers_from_pipeline(pipeline_path='src/models/preprocessing_pipeline.pkl', 
+def get_scalers_from_pipeline(pipeline_path='services/compress/models/preprocessing_pipeline.pkl', 
                              verbose=True, logging_enabled=True):
     """
     Load preprocessing pipeline and extract individual components with verbosity control.
@@ -174,15 +175,19 @@ def get_scalers_from_pipeline(pipeline_path='src/models/preprocessing_pipeline.p
             Returns (None, None, None, None, None) if loading fails
     """
     
-    # Import custom preprocessing classes needed for unpickling
+    # The preprocessing classes are now imported through utils module
+    # This ensures they're available when pickle tries to load the pipeline
     try:
-        from utils.data_preprocessing import (
-            ColumnDropper, VMAFScaler, TargetExtractor, CQScaler,
-            ResolutionTransformer, FeatureScaler, FrameRateTransformer
-        )
+        # Verify that the classes are accessible (this will fail if there's an import issue)
+        from utils import ColumnDropper, VMAFScaler, TargetExtractor, CQScaler, ResolutionTransformer, FeatureScaler, FrameRateTransformer
+        
+        if logging_enabled:
+            print(f"✅ Preprocessing classes are available through utils module")
+            
     except ImportError as e:
         if logging_enabled:
-            print(f"❌ ERROR: Could not import preprocessing classes: {e}")
+            print(f"❌ ERROR: Could not access preprocessing classes: {e}")
+            print(f"   💡 Check that utils/__init__.py properly imports data_preprocessing classes")
         return None, None, None, None, None
     
     # Control verbose output with master logging flag
@@ -200,7 +205,7 @@ def get_scalers_from_pipeline(pipeline_path='src/models/preprocessing_pipeline.p
     if not os.path.exists(pipeline_path):
         if actual_verbose:
             print(f"❌ ERROR: Pipeline file not found at '{pipeline_path}'")
-            print(f"   💡 Suggestion: Ensure preprocessing pipeline exists in src/models/ directory")
+            print(f"   💡 Suggestion: Ensure preprocessing pipeline exists in /models/ directory")
         return None, None, None, None, None
     
     try:
@@ -208,8 +213,88 @@ def get_scalers_from_pipeline(pipeline_path='src/models/preprocessing_pipeline.p
         if actual_verbose:
             print(f"📖 Loading preprocessing pipeline from: {pipeline_path}")
             
-        with open(pipeline_path, 'rb') as f:
-            pipeline_obj = pickle.load(f)
+        # Load the pickled preprocessing pipeline with custom unpickler to handle module namespace issues
+        class CustomUnpickler(pickle.Unpickler):
+            def find_class(self, module, name):
+                # if actual_verbose:
+                #     print(f"   🔍 Unpickler looking for: {module}.{name}")
+                
+                # Handle the case where classes were defined in __main__ context
+                if module == '__main__':
+                    if actual_verbose:
+                        print(f"   🔧 Handling __main__ context for class: {name}")
+                    # Try to get the class from utils module
+                    try:
+                        from utils import ColumnDropper, VMAFScaler, TargetExtractor, CQScaler, ResolutionTransformer, FeatureScaler, FrameRateTransformer
+                        class_map = {
+                            'ColumnDropper': ColumnDropper,
+                            'VMAFScaler': VMAFScaler,
+                            'TargetExtractor': TargetExtractor,
+                            'CQScaler': CQScaler,
+                            'ResolutionTransformer': ResolutionTransformer,
+                            'FeatureScaler': FeatureScaler,
+                            'FrameRateTransformer': FrameRateTransformer
+                        }
+                        if name in class_map:
+                            if actual_verbose:
+                                print(f"   ✅ Found {name} in utils module")
+                            return class_map[name]
+                        else:
+                            if actual_verbose:
+                                print(f"   ❌ Class {name} not found in utils module")
+                    except ImportError as e:
+                        if actual_verbose:
+                            print(f"   ❌ Failed to import from utils: {e}")
+                
+                # Handle utils.data_preprocessing module context
+                elif module == 'utils.data_preprocessing':
+                    if actual_verbose:
+                        print(f"   🔧 Handling utils.data_preprocessing context for class: {name}")
+                    try:
+                        from utils import ColumnDropper, VMAFScaler, TargetExtractor, CQScaler, ResolutionTransformer, FeatureScaler, FrameRateTransformer
+                        class_map = {
+                            'ColumnDropper': ColumnDropper,
+                            'VMAFScaler': VMAFScaler,
+                            'TargetExtractor': TargetExtractor,
+                            'CQScaler': CQScaler,
+                            'ResolutionTransformer': ResolutionTransformer,
+                            'FeatureScaler': FeatureScaler,
+                            'FrameRateTransformer': FrameRateTransformer
+                        }
+                        if name in class_map:
+                            if actual_verbose:
+                                print(f"   ✅ Found {name} in utils module")
+                            return class_map[name]
+                        else:
+                            if actual_verbose:
+                                print(f"   ❌ Class {name} not found in utils module")
+                    except ImportError as e:
+                        if actual_verbose:
+                            print(f"   ❌ Failed to import from utils: {e}")
+                
+                # For other modules, use the default behavior
+                if actual_verbose:
+                    print(f"   🔧 Using default unpickler for {module}.{name}")
+                return super().find_class(module, name)
+        
+        # Try to load with custom unpickler
+        try:
+            with open(pipeline_path, 'rb') as f:
+                unpickler = CustomUnpickler(f)
+                pipeline_obj = unpickler.load()
+            if actual_verbose:
+                print(f"   ✅ Pipeline loaded with custom unpickler")
+        except Exception as e:
+            if actual_verbose:
+                print(f"   ❌ Custom unpickler failed: {e}")
+                print(f"   🔧 Attempting to load with standard pickle...")
+            
+            # Fallback to standard pickle loading
+            with open(pipeline_path, 'rb') as f:
+                pipeline_obj = pickle.load(f)
+            
+            if actual_verbose:
+                print(f"   ✅ Pipeline loaded with standard pickle")
             
         if actual_verbose:
             print(f"✅ Pipeline loaded successfully")
@@ -428,8 +513,8 @@ def load_encoding_resources(config, logging_enabled=True):
     """
     # Get model file paths from config
     model_paths = config.get('model_paths', {})
-    scene_model_path = model_paths.get('scene_classifier_model', "../models/scene_classifier_model.pth")
-    preprocessing_pipeline_path = model_paths.get('preprocessing_pipeline', "src/models/preprocessing_pipeline.pkl")
+    scene_model_path = model_paths.get('scene_classifier_model', "services/compress/models/scene_classifier_model.pth")
+    preprocessing_pipeline_path = model_paths.get('preprocessing_pipeline', "services/compress/models/preprocessing_pipeline.pkl")
     
     # Choose GPU if available, otherwise use CPU
     device = 'cuda' if torch.cuda.is_available() else 'cpu'
@@ -761,6 +846,11 @@ def ai_encoding(scene_metadata, config, resources, target_vmaf=None, target_qual
             except Exception:
                 scene_duration = 1.0
 
+    # Check for very short scenes that might cause stuttering
+    is_very_short_scene = scene_duration < 1.0
+    if is_very_short_scene and logging_enabled:
+        print(f"   ⚠️ Very short scene detected ({scene_duration:.1f}s) - using special handling")
+
     # Get original video information for codec selection
     original_video_metadata = scene_metadata.get('original_video_metadata', {})
     
@@ -791,6 +881,9 @@ def ai_encoding(scene_metadata, config, resources, target_vmaf=None, target_qual
     target_vmaf = safe_float(target_vmaf or original_video_metadata.get('target_vmaf') or 
                            config.get('video_processing', {}).get('target_vmaf', 0.0), 0.0)
 
+
+    print("original video metadata", original_video_metadata)
+
     # Choose encoding codec (prioritize config over auto-detection)
     original_codec = original_video_metadata.get('original_codec', 'unknown')
     target_codec_from_part1 = original_video_metadata.get('target_codec', 'auto')
@@ -931,10 +1024,17 @@ def ai_encoding(scene_metadata, config, resources, target_vmaf=None, target_qual
     }
 
     # ENCODING
-    output_scene_path = os.path.join(
-        temp_dir,
-        f"encoded_scene_{scene_number:03d}_{start_time:.0f}s-{end_time:.0f}s_{codec.lower()}.mp4"
-    )
+    # Use more precise filename for very short scenes
+    if scene_duration < 1.0:
+        output_scene_path = os.path.join(
+            temp_dir,
+            f"encoded_scene_{scene_number:03d}_{start_time:.1f}s-{end_time:.1f}s_{codec.lower()}.mp4"
+        )
+    else:
+        output_scene_path = os.path.join(
+            temp_dir,
+            f"encoded_scene_{scene_number:03d}_{start_time:.0f}s-{end_time:.0f}s_{codec.lower()}.mp4"
+        )
     os.makedirs(temp_dir, exist_ok=True)
 
     size_increase_protection = config.get('video_processing', {}).get('size_increase_protection', True)

services/compress/scene_detector.py

@@ -3,6 +3,7 @@ import json
 import subprocess
 from utils.fast_scene_detect import adaptive_scene_detection_check
 from utils.split_video_into_scenes import split_video_into_scenes
+from utils.video_utils import get_keyframes
 
 def scene_detection(video_metadata):
     """
@@ -56,7 +57,8 @@ def scene_detection(video_metadata):
     duration = video_metadata['duration']
     codec = video_metadata['codec']
     original_codec = video_metadata.get('original_codec', codec)
-    target_codec = video_metadata.get('target_codec', 'auto')  # ✅ NEW: Extract target codec
+    target_codec = video_metadata.get('target_codec', 'auto') 
+    print("***********target_codec", target_codec) # ✅ NEW: Extract target codec
     was_reencoded = video_metadata.get('was_reencoded', False)
     target_vmaf = video_metadata.get('target_vmaf', 93.0)
     target_quality = video_metadata.get('target_quality', 'Medium')
@@ -82,33 +84,72 @@ def scene_detection(video_metadata):
             'duration': duration,
             'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
         }]
+    scene_config = config.get('scene_detection', {})
+    detection_mode = scene_config.get('mode', 'adaptive').lower()
+    
+    print(f"   🔍 Scene detection mode: {detection_mode}")
+    
+    if detection_mode == 'force_time_based':
+        # Force time-based splitting regardless of content
+        force_duration = scene_config.get('force_time_based_duration', 10)
+        print(f"   ⏰ Forcing time-based splitting with {force_duration}s segments")
+        return force_time_based_splitting(video_path, temp_dir, force_duration, duration, video_metadata)
+    
+    elif detection_mode == 'adaptive':
+        # Original adaptive scene detection logic
+        print(f"   🔍 Running adaptive scene detection...")
+        has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
+            video_path, duration, logging_enabled=True
+        )
 
-    # SCENE DETECTION: Check for multiple scenes
-    print(f"   🔍 Running adaptive scene detection...")
-    has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
-        video_path, duration, logging_enabled=True
-    )
-
-    if has_multiple_scenes:
-        print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
-        scene_files, _ = split_video_into_scenes(video_path, temp_dir)
-        
-        if scene_files and len(scene_files) > 1:
-            print(f"   📄 Created {len(scene_files)} scene files")
-            return create_scene_metadata_from_files(scene_files, video_metadata, config)
+        if has_multiple_scenes:
+            print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
+            scene_files, _ = split_video_into_scenes(video_path, temp_dir)
+            
+            if scene_files and len(scene_files) > 1:
+                print(f"   📄 Created {len(scene_files)} scene files")
+                return create_scene_metadata_from_files(scene_files, video_metadata, config)
+            else:
+                print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
         else:
-            print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
-            return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+            print(f"   📏 Single scene detected.")
+            enable_fallback = scene_config.get('enable_time_based_fallback', False)
+            
+            if enable_fallback:
+                print(f"   🔄 Time-based fallback enabled. Attempting time-based scene splitting...")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+            else:
+                print(f"   ✅ Using single scene (time-based fallback disabled)")
+                return [{
+                    'path': video_path,
+                    'scene_number': 1,
+                    'start_time': 0.0,
+                    'end_time': duration,
+                    'duration': duration,
+                    'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
+                }]
+    
     else:
-        print(f"   📏 Single scene detected.")
-        scene_config = config.get('scene_detection', {})
-        enable_fallback = scene_config.get('enable_time_based_fallback', False)
-        
-        if enable_fallback:
-            print(f"   🔄 Time-based fallback enabled. Attempting time-based scene splitting...")
-            return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
+        print(f"   ❌ Unknown scene detection mode: {detection_mode}. Falling back to adaptive.")
+        # Fallback to adaptive mode
+        print(f"   🔍 Running adaptive scene detection...")
+        has_multiple_scenes, estimated_scenes, _ = adaptive_scene_detection_check(
+            video_path, duration, logging_enabled=True
+        )
+
+        if has_multiple_scenes:
+            print(f"   ✅ Multiple scenes detected ({estimated_scenes} estimated). Proceeding with full scene splitting.")
+            scene_files, _ = split_video_into_scenes(video_path, temp_dir)
+            
+            if scene_files and len(scene_files) > 1:
+                print(f"   📄 Created {len(scene_files)} scene files")
+                return create_scene_metadata_from_files(scene_files, video_metadata, config)
+            else:
+                print(f"   ⚠️ Scene splitting failed or found only 1 scene. Falling back to time-based splitting.")
+                return time_based_splitting(video_path, temp_dir, config, duration, video_metadata)
         else:
-            print(f"   ✅ Using single scene (time-based fallback disabled)")
+            print(f"   📏 Single scene detected. Using single scene.")
             return [{
                 'path': video_path,
                 'scene_number': 1,
@@ -118,6 +159,208 @@ def scene_detection(video_metadata):
                 'original_video_metadata': video_metadata  # ✅ Pass complete metadata including target_codec
             }]
 
+def force_time_based_splitting(video_path, temp_dir, segment_duration, total_duration, original_metadata, logging_enabled=True):
+    """
+    Force time-based splitting with keyframe-aligned boundaries to prevent stuttering.
+    
+    Args:
+        video_path (str): Path to the video file
+        temp_dir (str): Temporary directory for scene files
+        segment_duration (float): Target duration for each segment in seconds
+        total_duration (float): Total video duration in seconds
+        original_metadata (dict): Complete metadata from Part 1
+        logging_enabled (bool): Enable detailed logging
+    
+    Returns:
+        list: List of scene metadata dictionaries
+    """
+    if logging_enabled:
+        print(f"\n--- Force time-based splitting: {segment_duration}s segments ---")
+        print(f"   🎥 Source codec: {original_metadata.get('codec', 'unknown')}")
+        print(f"   ⏱️ Total duration: {total_duration:.1f}s")
+        print(f"   ✂️ Segment duration: {segment_duration}s")
+    
+    # Get keyframes for precise cutting
+    try:
+        keyframes = get_keyframes(video_path)
+        if logging_enabled:
+            print(f"   🔑 Found {len(keyframes)} keyframes for alignment")
+    except Exception as e:
+        # Try alternative import path
+        try:
+            keyframes = get_keyframes(video_path)
+            if logging_enabled:
+                print(f"   🔑 Found {len(keyframes)} keyframes for alignment")
+        except Exception as e2:
+            keyframes = []
+            if logging_enabled:
+                print(f"   ⚠️ Could not get keyframes: {e}")
+                print(f"   📐 Using time-based boundaries (may cause stuttering)")
+    
+    # Calculate scene boundaries, preferring keyframe alignment
+    scene_boundaries = []
+    current_start = 0
+    
+    while current_start < total_duration:
+        target_end = min(current_start + segment_duration, total_duration)
+        
+        # If we have keyframes, find the closest keyframe to target_end
+        if keyframes and target_end < total_duration:
+            # Find keyframe closest to target_end
+            closest_keyframe = min(keyframes, key=lambda kf: abs(kf - target_end))
+            
+            # Only use keyframe if it's reasonably close (within 2 seconds)
+            if abs(closest_keyframe - target_end) <= 2.0 and closest_keyframe > current_start:
+                scene_end = closest_keyframe
+            else:
+                scene_end = target_end
+        else:
+            scene_end = target_end
+        
+        # Handle very short final segments (< 2 seconds) by merging with previous scene
+        remaining_duration = total_duration - scene_end
+        if remaining_duration > 0 and remaining_duration < 2.0 and scene_boundaries:
+            # Extend current scene to end of video instead of creating tiny final scene
+            scene_end = total_duration
+            if logging_enabled:
+                print(f"   🔧 Merging final {remaining_duration:.1f}s with current scene to avoid stuttering")
+        
+        scene_boundaries.append((current_start, scene_end))
+        current_start = scene_end
+    
+    if logging_enabled:
+        print(f"   📊 Will create {len(scene_boundaries)} segments:")
+        for i, (start, end) in enumerate(scene_boundaries):
+            print(f"      Segment {i+1}: {start:.1f}s - {end:.1f}s (duration: {end-start:.1f}s)")
+    
+    # If only one scene, return original video
+    if len(scene_boundaries) <= 1:
+        if logging_enabled:
+            print(f"   📏 Video shorter than segment duration, using as single scene")
+        return [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
+    
+    # Create output directory
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    # Choose scene file extension based on source codec
+    source_codec = original_metadata.get('codec', 'h264').lower()
+    if source_codec in ['ffv1', 'prores', 'dnxhd']:
+        scene_extension = '.mkv'  # Use MKV for lossless codecs
+    elif source_codec in ['av1', 'vp9']:
+        scene_extension = '.webm'  # Use WebM for modern codecs
+    else:
+        scene_extension = '.mp4'  # Default to MP4 for compatibility
+    
+    if logging_enabled:
+        print(f"   📦 Using {scene_extension} container for {source_codec} codec")
+    
+    # Split video and create metadata
+    scene_metadata_list = []
+    
+    for i, (start_time_sec, end_time_sec) in enumerate(scene_boundaries):
+        scene_number = i + 1
+        scene_filename = f"scene_{scene_number:03d}_time_{start_time_sec:.0f}s{scene_extension}"
+        scene_path = os.path.join(temp_dir, scene_filename)
+        
+        duration_scene = end_time_sec - start_time_sec
+        
+        # FFmpeg command with precise cutting to prevent frame alignment issues
+        # Use input seeking for efficiency and output seeking for precision
+        cmd = [
+            'ffmpeg', '-y',
+            '-ss', str(start_time_sec),     # Input seeking for efficiency
+            '-i', video_path,
+            '-t', str(duration_scene),      # Precise duration
+            '-avoid_negative_ts', 'make_zero',
+            '-fflags', '+genpts',           # Generate presentation timestamps
+            '-map', '0:v:0',                # Map video stream explicitly
+            '-map', '0:a?',                 # Map audio if exists (optional)
+        ]
+        
+        # Add codec-specific format options before encoding
+        format_options = []
+        if source_codec == 'ffv1':
+            format_options.extend(['-f', 'matroska'])
+        elif source_codec in ['av1', 'vp9']:
+            format_options.extend(['-f', 'webm'])
+        
+        # Always use fast re-encoding to ensure clean cuts and prevent stuttering
+        # Stream copy can cause artifacts at scene boundaries even with keyframe alignment
+        cmd.extend([
+            '-c:v', 'libx264',
+            '-preset', 'ultrafast',
+            '-crf', '18',
+            '-c:a', 'copy'
+        ])
+        cmd.extend(format_options)  # Add format options
+        cmd.append(scene_path)      # Add output file path
+        
+        if logging_enabled:
+            print(f"   ✂️ Creating segment {scene_number}: {scene_filename}")
+        
+        try:
+            # Execute FFmpeg command
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+            
+            if result.returncode == 0:
+                # Verify file exists and has content
+                if os.path.exists(scene_path) and os.path.getsize(scene_path) > 0:
+                    file_size_mb = os.path.getsize(scene_path) / (1024 * 1024)
+                    
+                    scene_metadata = {
+                        'path': scene_path,
+                        'scene_number': scene_number,
+                        'start_time': start_time_sec,
+                        'end_time': end_time_sec,
+                        'duration': duration_scene,
+                        'scene_codec': source_codec,
+                        'scene_container': scene_extension[1:],  # Remove the dot
+                        'file_size_mb': file_size_mb,
+                        'scene_detection_method': 'force_time_based',
+                        'original_video_metadata': original_metadata
+                    }
+                    scene_metadata_list.append(scene_metadata)
+                    
+                    if logging_enabled:
+                        print(f"      ✅ Created: {scene_filename} ({file_size_mb:.1f} MB)")
+                else:
+                    if logging_enabled:
+                        print(f"      ❌ Failed: {scene_filename} (file not created or empty)")
+            else:
+                if logging_enabled:
+                    print(f"      ❌ FFmpeg failed for {scene_filename}: {result.stderr}")
+                
+        except subprocess.TimeoutExpired:
+            if logging_enabled:
+                print(f"      ❌ Timeout creating {scene_filename}")
+        except Exception as e:
+            if logging_enabled:
+                print(f"      ❌ Error creating {scene_filename}: {e}")
+    
+    if not scene_metadata_list:
+        if logging_enabled:
+            print(f"   ❌ All segment creation failed, using original video as single scene")
+        return [{
+            'path': video_path,
+            'scene_number': 1,
+            'start_time': 0.0,
+            'end_time': total_duration,
+            'duration': total_duration,
+            'original_video_metadata': original_metadata
+        }]
+    
+    if logging_enabled:
+        print(f"   ✅ Successfully created {len(scene_metadata_list)} time-based segments")
+    
+    return scene_metadata_list
+
 def time_based_splitting(video_path, temp_dir, config, duration, original_metadata, logging_enabled=True):
     """
     Split video into scenes and return metadata for each scene.

services/compress/server.py

@@ -156,8 +156,8 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
         print("⚠️ Config file not found, using default configuration")
         config = {
             'directories': {
-                'temp_dir': str(Path('./videos/temp_scenes').absolute()),
-                'output_dir': str(output_dir_path)
+                'temp_dir': './videos/temp_scenes',
+                'output_dir': './output'
             },
             'video_processing': {
                 'SHORT_VIDEO_THRESHOLD': 20,
@@ -166,7 +166,31 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
                 'size_increase_protection': True,
                 'conservative_cq_adjustment': 2,
                 'max_output_size_ratio': 1.15,
-                'max_encoding_retries': 2
+                'max_encoding_retries': 2,
+                # ✅ BASIC MINER: Quality-based CQ lookup tables
+                'basic_cq_lookup_by_quality': {
+                    'High': {     # Target VMAF: 95
+                        'animation': 22,
+                        'low-action': 20,
+                        'medium-action': 18,
+                        'high-action': 16,
+                        'default': 19
+                    },
+                    'Medium': {   # Target VMAF: 93
+                        'animation': 25,
+                        'low-action': 23,
+                        'medium-action': 21,
+                        'high-action': 19,
+                        'default': 22
+                    },
+                    'Low': {      # Target VMAF: 90
+                        'animation': 28,
+                        'low-action': 26,
+                        'medium-action': 24,
+                        'high-action': 22,
+                        'default': 25
+                    }
+                },
             },
             'scene_detection': {
                 'enable_time_based_fallback': True,
@@ -181,12 +205,17 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
             'output_settings': {
                 'save_individual_scene_reports': True,
                 'save_comprehensive_report': True
+            },
+            'model_paths': {
+                'scene_classifier_model': 'services/compress/models/scene_classifier_model.pth'
             }
         }
     
-    # Update output directory from arguments
     config['directories']['output_dir'] = str(output_dir_path)
-    
+    if 'video_processing' not in config:
+        config['video_processing'] = {}
+    config['video_processing']['target_quality'] = target_quality
+
     # Create temp directory if it doesn't exist
     temp_dir = Path(config['directories']['temp_dir'])
     temp_dir.mkdir(parents=True, exist_ok=True)
@@ -208,6 +237,8 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
         max_duration=max_duration,
         output_dir=output_dir_path
     )
+
+    print("part1_result", part1_result)
     
     part1_time = time.time() - part1_start_time
     
@@ -231,6 +262,8 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
     part2_start_time = time.time()
     
     scenes_metadata = scene_detection(part1_result)
+
+    print("scenes_metadata", scenes_metadata)
     
     part2_time = time.time() - part2_start_time
     
@@ -255,17 +288,34 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
     if total_scene_size > 0:
         print(f"   📊 Total scene files: {total_scene_size:.1f} MB")
 
-    # ✅ PART 3: AI Encoding (Multiple Scene Processing)
-    print(f"\n🤖 === Part 3: AI Encoding ===")
+    # ✅ PART 3: AI Encoding (BASIC MINER - Multiple Scene Processing)
+    print(f"\n🧠 === Part 3: AI Encoding (Basic Miner) ===")
     part3_start_time = time.time()
     
+    print(f"   🔧 Loading basic AI models and resources...")
+    print(f"   📋 Using quality-based CQ lookup tables for {target_quality} quality")
+    print(f"   🎯 Target Quality Level: {target_quality}")
+    
+    # Display CQ ranges for selected quality level
+    quality_info = {
+        'High': {'vmaf': 95, 'cq_range': '16-22'},
+        'Medium': {'vmaf': 93, 'cq_range': '19-25'},
+        'Low': {'vmaf': 90, 'cq_range': '22-28'}
+    }
+    
+    if target_quality in quality_info:
+        info = quality_info[target_quality]
+        print(f"   🎚️ CQ Range for {target_quality}: {info['cq_range']} (Target VMAF: {info['vmaf']})")
+    
+    
     print(f"   🔧 Loading AI models and resources...")
     
     try:
         resources = load_encoding_resources(config, logging_enabled=True)
-        print(f"   ✅ AI resources loaded successfully")
+        print(f"   ✅ Basic AI resources loaded successfully")
+        print(f"   🧠 Mode: Scene classification + CQ lookup table")
     except Exception as e:
-        print(f"   ❌ Failed to load AI resources: {e}")
+        print(f"   ❌ Failed to load GGG AI resources: {e}")
         return False
     
     # Process each scene individually
@@ -275,7 +325,7 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
     total_input_size = 0
     total_output_size = 0
     
-    print(f"\n   📊 Processing {len(scenes_metadata)} scenes individually...")
+    print(f"\n   📊 Processing {len(scenes_metadata)} scenes with basic AI approach...")
     
     for i, scene_metadata in enumerate(scenes_metadata):
         scene_number = scene_metadata['scene_number']
@@ -284,17 +334,21 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
         
         print(f"\n   🎬 Scene {scene_number}/{len(scenes_metadata)}: {os.path.basename(scene_path)}")
         print(f"      ⏱️ Duration: {scene_duration:.1f}s")
-        print(f"      🎯 Target VMAF: {scene_metadata['original_video_metadata']['target_vmaf']}")
+        # Show both selected quality level and any indicative target VMAF (if present)
+        indicative_vmaf = scene_metadata['original_video_metadata'].get('target_vmaf')
+        print(f"      🎯 Target Quality: {target_quality}" + (f" (VMAF≈{indicative_vmaf})" if indicative_vmaf else ""))
+        print(f"      🧠 Method: Basic scene classification + CQ lookup")
         
         scene_start_time = time.time()
         
         try:
-            # ✅ SINGLE SCENE PROCESSING: Call Part 3 for individual scene
+            # ✅ BASIC MINER: Call Part 3 for individual scene using basic approach
             encoded_path, scene_data = ai_encoding(
                 scene_metadata=scene_metadata,
                 config=config,
                 resources=resources,
-                target_vmaf=None,  # Let it use the target from original video metadata
+                target_vmaf=None,  # VMAF not used in basic mode; we pass quality level instead
+                target_quality_level=target_quality,
                 logging_enabled=True
             )
             
@@ -313,7 +367,9 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
                 print(f"         📁 Output: {os.path.basename(encoded_path)}")
                 print(f"         📊 Size: {input_size_mb:.1f} MB → {size_mb:.1f} MB ({compression:+.1f}% compression)")
                 print(f"         🎭 Scene type: {scene_data.get('scene_type', 'unknown')}")
-                print(f"         🎚️ CQ used: {scene_data.get('final_adjusted_cq', 'unknown')}")
+                print(f"         � Quality: {scene_data.get('target_quality_level', target_quality)}")
+                print(f"         �🎚️ CQ used: {scene_data.get('base_cq_for_quality', 'N/A')} → {scene_data.get('final_adjusted_cq', 'unknown')} (after adjustment)")
+                print(f"         📋 Method: Quality-based lookup table CQ selection")
                 print(f"         ⏱️ Processing: {scene_processing_time:.1f}s")
                 
                 # Update scene metadata with encoded path for Part 4
@@ -357,11 +413,13 @@ def video_compressor(input_file, target_quality='Medium', max_duration=3600, out
     part3_time = time.time() - part3_start_time
     
     # ✅ PART 3 SUMMARY
-    print(f"\n   📊 Part 3 Processing Summary:")
+    print(f"\n   📊 Part 3 Basic Processing Summary:")
     print(f"      ✅ Successful encodings: {successful_encodings}")
     print(f"      ❌ Failed encodings: {failed_encodings}")
     print(f"      📈 Success rate: {successful_encodings/len(scenes_metadata)*100:.1f}%")
     print(f"      ⏱️ Total processing time: {part3_time:.1f}s")
+    print(f"      🎯 Quality Level: {target_quality}")
+    print(f"      🧠 AI Method: Scene classification + quality-based CQ lookup")
     
     if successful_encodings == 0:
         print("❌ Part 3 failed completely. No scenes were encoded. Pipeline terminated.")
@@ -532,13 +590,13 @@ def test_video_compression(video_path: str):
 if __name__ == "__main__":
     import uvicorn
 
-    logger.info("Starting video compressor server")
-    logger.info(f"Video compressor server running on http://{CONFIG.video_compressor.host}:{CONFIG.video_compressor.port}")
+    # logger.info("Starting video compressor server")
+    # logger.info(f"Video compressor server running on http://{CONFIG.video_compressor.host}:{CONFIG.video_compressor.port}")
 
-    uvicorn.run(app, host=CONFIG.video_compressor.host, port=CONFIG.video_compressor.port)
+    # uvicorn.run(app, host=CONFIG.video_compressor.host, port=CONFIG.video_compressor.port)
 
-    # result = test_video_compression('test1.mp4')
-    # print(result)
+    result = test_video_compression('test1.mp4')
+    print(result)
 
 
     #python services/compress/server.py

services/compress/utils/__init__.py

@@ -9,6 +9,12 @@ from .processing_utils import (
 )
 from .fast_scene_detect import adaptive_scene_detection_check, fast_scene_detection_check, motion_based_scene_check
 
+# Import preprocessing classes to make them available for pickle loading
+from .data_preprocessing import (
+    ColumnDropper, VMAFScaler, TargetExtractor, CQScaler,
+    ResolutionTransformer, FeatureScaler, FrameRateTransformer
+)
+
 __all__ = [
     'monitor_memory_usage', 'ProgressTracker',
     'calculate_perceptual_contrast', 'get_video_duration', 
@@ -21,5 +27,8 @@ __all__ = [
     'should_skip_encoding',
     'create_summary_and_save', 'cleanup_temporary_items',
     'adaptive_scene_detection_check', 'fast_scene_detection_check', 'motion_based_scene_check',
-    'VideoProcessingLogger'
+    'VideoProcessingLogger',
+    # Add preprocessing classes to make them available
+    'ColumnDropper', 'VMAFScaler', 'TargetExtractor', 'CQScaler',
+    'ResolutionTransformer', 'FeatureScaler', 'FrameRateTransformer'
 ]

services/compress/utils/encode_video.py

@@ -8,14 +8,8 @@ import time
 import traceback
 import numpy as np
 
-try:
-    from .encoder_configs import ENCODER_SETTINGS, SCENE_SPECIFIC_PARAMS, MODEL_CQ_REFERENCE_CODEC, QUALITY_MAPPING_ANCHORS
-except ImportError:
-    print("Error: Could not import configurations from 'encoder_configs.py'. Make sure the file exists in the 'src' directory.")
-    ENCODER_SETTINGS = {}
-    SCENE_SPECIFIC_PARAMS = {}
-    MODEL_CQ_REFERENCE_CODEC = ""
-    QUALITY_MAPPING_ANCHORS = {}
+from .encoder_configs import ENCODER_SETTINGS, SCENE_SPECIFIC_PARAMS, MODEL_CQ_REFERENCE_CODEC, QUALITY_MAPPING_ANCHORS
+
 
 
 def get_contrast_optimized_params(scene_type, contrast_value, codec):
@@ -61,7 +55,7 @@ def get_contrast_optimized_params(scene_type, contrast_value, codec):
             params['temporal-aq'] = 0  # Default
     
     # --- x264 ---
-    elif codec == "H264" or "libx264" in codec:
+    elif codec == "libx264":  # ✅ FIXED: Use standardized codec name
         # AQ mode and strength adjustments
         if contrast_category == "high":
             params['aq-mode'] = 2  # Variance AQ with auto-variance
@@ -74,7 +68,7 @@ def get_contrast_optimized_params(scene_type, contrast_value, codec):
             params['aq-strength'] = 1.0  # Default
     
     # --- x265 ---
-    elif codec == "hevc" or "libx265" in codec:
+    elif codec == "libx265":  # ✅ FIXED: Use standardized codec name
         # AQ mode and strength adjustments
         if contrast_category == "high":
             params['aq-mode'] = 3  # Auto-variance AQ with more aggressive bias
@@ -86,47 +80,59 @@ def get_contrast_optimized_params(scene_type, contrast_value, codec):
             params['aq-mode'] = 2
             params['aq-strength'] = 1.0  # Default
     
-    # --- libaom-av1 ---
-    elif "libaom-av1" in codec or codec == "av1_fallback":
+    # --- SVT-AV1 ---
+    elif codec == "libsvtav1":  # ✅ FIXED: Use standardized codec name
+        # SVT-AV1 doesn't have enable-hdr or aq-mode options
+        # Use CRF adjustments and available parameters instead
         if contrast_category == "high":
-            params['aq-mode'] = 2  # Variance-based adaptive quantization
-            params['deltaq-mode'] = 3  # Perceptual deltaq mode
+            # For high contrast, we can rely on the encoder's default behavior
+            # or use dolbyvision if HDR content is detected
+            pass  # SVT-AV1 handles high contrast well by default
         elif contrast_category == "low":
-            params['aq-mode'] = 1  # Default adaptive quantization
-            params['deltaq-mode'] = 0  # Disabled
+            # For low contrast, we might want a slightly lower CRF for better quality
+            pass  # This would be handled at the CRF level, not here
         else:
-            params['aq-mode'] = 1
-            params['deltaq-mode'] = 0
+            pass  # Default SVT-AV1 behavior is generally good
     
-    # --- SVT-AV1 ---
-    elif "libsvtav1" in codec or codec == "av1_optimized":
-        # params for SVT-AV1
+    # --- VP9 ---
+    elif codec == "libvpx_vp9":  # ✅ FIXED: Use standardized codec name (underscore)
+        # VP9 adaptive quantization and quality parameters
         if contrast_category == "high":
-            params['aq-mode'] = 2
+            params['aq-mode'] = 3  # Complexity-based AQ for high contrast
+            params['arnr-maxframes'] = 7  # More noise reduction frames
+            params['arnr-strength'] = 5  # Higher noise reduction strength
         elif contrast_category == "low":
-            params['aq-mode'] = 1
+            params['aq-mode'] = 2  # Variance-based AQ for low contrast  
+            params['arnr-maxframes'] = 5  # Standard noise reduction
+            params['arnr-strength'] = 3  # Moderate noise reduction
         else:
-            params['aq-mode'] = 1
+            params['aq-mode'] = 1  # Variance-based AQ (default)
+            params['arnr-maxframes'] = 7  # Standard noise reduction
+            params['arnr-strength'] = 4  # Standard noise reduction strength
+        
+        # VP9-specific quality settings
+        params['tune'] = 'psnr'  # Optimize for quality
+        params['auto-alt-ref'] = 1  # Enable alternate reference frames
     
     # --- Scene-specific contrast adjustments ---
     # For text content, adjust parameters further based on contrast
     if scene_type == 'Screen Content / Text':
         if contrast_category == "high":
             params['sharpness'] = 0  # Preserve sharpness
-            if "NVENC" in codec:
+            if "_nvenc" in codec:  # ✅ FIXED: lowercase nvenc
                 params['rc-lookahead'] = 20  # More lookahead for complex text
         elif contrast_category == "low":
             params['sharpness'] = 1  # Some sharpening for low contrast text
-            if "NVENC" in codec:
+            if "_nvenc" in codec:  # ✅ FIXED: lowercase nvenc
                 params['rc-lookahead'] = 8  # Less lookahead needed
     
     # For faces, special handling based on contrast
     elif scene_type == 'Faces / People':
         if contrast_category == "high":
-            if "NVENC" in codec:
+            if "_nvenc" in codec:  # ✅ FIXED: lowercase nvenc
                 params['rc-lookahead'] = 20  # More lookahead for dramatic lighting
         else:
-            if "NVENC" in codec:
+            if "_nvenc" in codec:  # ✅ FIXED: lowercase nvenc
                 params['rc-lookahead'] = 15  # Standard lookahead
     
     return params
@@ -138,13 +144,27 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
     
     Note: encoding decisions should be made BEFORE calling this function.
     
-    """
+    Args:
+        input_path (str): Path to input video file
+        output_path (str): Path for output video file
+        codec (str): Video codec to use (e.g., 'av1_nvenc', 'libx264')
+        rate (int/float, optional): Quality parameter (CQ value from model)
+        max_bit_rate (str, optional): Maximum bitrate (e.g., '5000k', '10m')
+        preset (str, optional): Encoder preset override
+        scene_type (str, optional): Scene classification for optimization
+        contrast_value (float, optional): Perceptual contrast (0.0-1.0)
+        logging_enabled (bool): Enable detailed logging
         
+    Returns:
+        tuple: (encoding_results_log, encoding_time) or (None, None) on failure
+    """
+    
     if logging_enabled:
         print(f"Encoding video using codec: {codec}, scene: {scene_type}, model_predicted_rate: {rate}")
         if contrast_value is not None:
             print(f"Using contrast value: {contrast_value:.2f}")
 
+    # Get base encoder settings
     base_settings = ENCODER_SETTINGS.get(codec)
     if not base_settings:
         print(f"Error: Codec settings for '{codec}' not found in ENCODER_SETTINGS.")
@@ -157,12 +177,11 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
     # 2. Apply scene-specific overrides if scene_type is provided
     if scene_type:
         codec_scene_params = SCENE_SPECIFIC_PARAMS.get(codec, {})
-        # Get params for the specific scene, fallback to 'other', then empty dict
         scene_params = codec_scene_params.get(scene_type, codec_scene_params.get('other', {}))
         if scene_params:
             if logging_enabled:
                 print(f"Applying scene-specific params for '{scene_type}': {scene_params}")
-            current_settings.update(scene_params) # Update base with scene specifics
+            current_settings.update(scene_params)
 
     # 3. Apply contrast-specific overrides if contrast_value is provided
     if contrast_value is not None:
@@ -172,39 +191,47 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
                 print(f"Applying contrast-specific params: {contrast_params}")
             current_settings.update(contrast_params)
 
-    # 4. Apply function arguments as highest priority overrides
-    # Apply 'rate' (which is the model_predicted_av1_cq)
+    # 4. Apply rate (CQ) parameter with codec-specific handling
     if rate is not None:
-        model_predicted_ref_cq = int(rate) # This is the CQ from find_optimal_cq, on MODEL_CQ_REFERENCE_CODEC's scale
-
+        model_predicted_ref_cq = int(rate)
+        
         if codec == MODEL_CQ_REFERENCE_CODEC:
             # Use the predicted CQ directly for the reference codec
-            if 'cq' in current_settings: # Assuming reference codec uses 'cq'
+            if 'cq' in current_settings:
                 current_settings['cq'] = model_predicted_ref_cq
-                if 'crf' in current_settings: del current_settings['crf']
-                # Ensure rate control is set correctly for CQ (e.g., constqp for NVENC)
-                if codec.endswith("_NVENC") or codec.endswith("_QSV") or 'videotoolbox' in codec: # Added videotoolbox
-                    current_settings['rc'] = 'constqp' # Or appropriate for videotoolbox if it differs
-                if logging_enabled: print(f"Applying model CQ directly for {MODEL_CQ_REFERENCE_CODEC}: {current_settings['cq']}")
-            elif 'crf' in current_settings: # If reference codec uses CRF (less common for AV1 model output)
-                current_settings['crf'] = model_predicted_ref_cq # Or a 1:1 mapping if scales are identical
-                if 'cq' in current_settings: del current_settings['cq']
-                if logging_enabled: print(f"Applying model CQ as CRF for {MODEL_CQ_REFERENCE_CODEC}: {current_settings['crf']}")
+                if 'crf' in current_settings: 
+                    del current_settings['crf']
+                # Use VBR for NVENC/QSV with CQ
+                if codec.endswith("_nvenc") or codec.endswith("_qsv"):
+                    current_settings['rc'] = 'vbr'
+                if 'qp' in current_settings: 
+                    del current_settings['qp']
+                if logging_enabled: 
+                    print(f"Applying model CQ directly for {MODEL_CQ_REFERENCE_CODEC}: {current_settings['cq']}")
+            elif 'crf' in current_settings:
+                current_settings['crf'] = model_predicted_ref_cq
+                if 'cq' in current_settings: 
+                    del current_settings['cq']
+                if logging_enabled: 
+                    print(f"Applying model CQ as CRF for {MODEL_CQ_REFERENCE_CODEC}: {current_settings['crf']}")
             else:
-                if logging_enabled: print(f"Warning: Neither 'cq' nor 'crf' in base settings for {MODEL_CQ_REFERENCE_CODEC}. Applying rate as 'crf'.")
+                if logging_enabled: 
+                    print(f"Warning: Neither 'cq' nor 'crf' in base settings for {MODEL_CQ_REFERENCE_CODEC}. Using CRF fallback.")
                 current_settings['crf'] = model_predicted_ref_cq
 
         elif codec in QUALITY_MAPPING_ANCHORS:
+            # Use quality mapping for non-reference codecs
             mapping_config = QUALITY_MAPPING_ANCHORS[codec]
             
             model_anchor_cqs = [p[0] for p in mapping_config['anchor_points']]
             target_anchor_params = [p[1] for p in mapping_config['anchor_points']]
 
-            # Interpolate
-            # Ensure model_predicted_ref_cq is within the model_ref_cq_range for stable interpolation
-            clamped_model_cq = np.clip(model_predicted_ref_cq, mapping_config['model_ref_cq_range'][0], mapping_config['model_ref_cq_range'][1])
+            # Interpolate with clamping
+            clamped_model_cq = np.clip(model_predicted_ref_cq, 
+                                     mapping_config['model_ref_cq_range'][0], 
+                                     mapping_config['model_ref_cq_range'][1])
             if logging_enabled and clamped_model_cq != model_predicted_ref_cq:
-                print(f"Clamped model_predicted_ref_cq from {model_predicted_ref_cq} to {clamped_model_cq} for interpolation based on model_ref_cq_range.")
+                print(f"Clamped model_predicted_ref_cq from {model_predicted_ref_cq} to {clamped_model_cq}")
 
             mapped_param_float = np.interp(clamped_model_cq, model_anchor_cqs, target_anchor_params)
             
@@ -216,101 +243,163 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
             target_param_type = mapping_config['target_param_type']
             if target_param_type == 'cq':
                 current_settings['cq'] = mapped_param_int
-                if 'crf' in current_settings: del current_settings['crf']
-                if codec.endswith("_NVENC") or codec.endswith("_QSV") or 'videotoolbox' in codec: # Added videotoolbox
-                    current_settings['rc'] = 'constqp'
-                if logging_enabled: print(f"Applying mapped CQ for {codec} from model ref CQ {model_predicted_ref_cq}: {current_settings['cq']}")
+                if 'crf' in current_settings: 
+                    del current_settings['crf']
+                # Use VBR for NVENC and QSV when using CQ
+                if codec.endswith("_nvenc") or codec.endswith("_qsv"):
+                    current_settings['rc'] = 'vbr'
+                if 'qp' in current_settings: 
+                    del current_settings['qp']
+                if logging_enabled: 
+                    print(f"Applying mapped CQ (VBR) for {codec} from model ref CQ {model_predicted_ref_cq}: {current_settings['cq']}")
             elif target_param_type == 'crf':
                 current_settings['crf'] = mapped_param_int
-                if 'cq' in current_settings: del current_settings['cq']
-                # Specific handling for codecs like libaom-av1 if they use CRF and need b:v=0
-                if base_settings.get("codec") == "libaom-av1" and 'b:v' in current_settings:
-                    current_settings['b:v'] = "0"
-                if logging_enabled: print(f"Applying mapped CRF for {codec} from model ref CQ {model_predicted_ref_cq}: {current_settings['crf']}")
+                if 'cq' in current_settings: 
+                    del current_settings['cq']
+                if logging_enabled: 
+                    print(f"Applying mapped CRF for {codec} from model ref CQ {model_predicted_ref_cq}: {current_settings['crf']}")
             else:
-                if logging_enabled: print(f"Warning: Unknown target_param_type '{target_param_type}' for {codec}. Using direct rate as fallback.")
-                if 'cq' in current_settings: current_settings['cq'] = model_predicted_ref_cq
-                elif 'crf' in current_settings: current_settings['crf'] = model_predicted_ref_cq
+                if logging_enabled: 
+                    print(f"Warning: Unknown target_param_type '{target_param_type}' for {codec}. Using fallback.")
+                if 'cq' in current_settings: 
+                    current_settings['cq'] = model_predicted_ref_cq
+                elif 'crf' in current_settings: 
+                    current_settings['crf'] = model_predicted_ref_cq
 
         else:
+            # Fallback for codecs without mapping
             if logging_enabled:
-                print(f"Warning: No quality mapping found for {codec} in QUALITY_MAPPING_ANCHORS. Applying model ref CQ {model_predicted_ref_cq} directly (may be inappropriate).")
-            # Fallback: apply the rate directly, trying 'cq' then 'crf'
+                print(f"Warning: No quality mapping found for {codec}. Applying model ref CQ {model_predicted_ref_cq} directly.")
             if 'cq' in current_settings:
-                 current_settings['cq'] = model_predicted_ref_cq
-                 if 'crf' in current_settings: del current_settings['crf']
-                 if codec.endswith("_NVENC") or codec.endswith("_QSV") or 'videotoolbox' in codec: current_settings['rc'] = 'constqp'
+                current_settings['cq'] = model_predicted_ref_cq
+                if 'crf' in current_settings: 
+                    del current_settings['crf']
+                # Use VBR for NVENC and QSV with CQ
+                if codec.endswith("_nvenc") or codec.endswith("_qsv"):
+                    current_settings['rc'] = 'vbr'
+                if 'qp' in current_settings: 
+                    del current_settings['qp']
             elif 'crf' in current_settings:
-                 current_settings['crf'] = model_predicted_ref_cq
-                 if 'cq' in current_settings: del current_settings['cq']
-                 if base_settings.get("codec") == "libaom-av1" and 'b:v' in current_settings: current_settings['b:v'] = "0"
-            else: # Fallback: Assume CRF if neither is defined in base settings
-                 current_settings['crf'] = model_predicted_ref_cq
-            if logging_enabled: print(f"Applied direct rate {model_predicted_ref_cq} to {codec}.")
-
-    # Apply 'preset' override if provided directly to function
+                current_settings['crf'] = model_predicted_ref_cq
+                if 'cq' in current_settings: 
+                    del current_settings['cq']
+            else:
+                current_settings['crf'] = model_predicted_ref_cq
+            if logging_enabled: 
+                print(f"Applied direct rate {model_predicted_ref_cq} to {codec}.")
+
+    # 5. Apply preset override if provided
     if preset is not None:
         current_settings['preset'] = preset
         if logging_enabled:
             print(f"Applying preset override: {preset}")
 
-    # Apply 'max_bit_rate' override (and calculate bufsize)
+    # 6. Apply max_bit_rate and enable VBR for NVENC/QSV when maxrate is provided
     if max_bit_rate is not None:
         current_settings['maxrate'] = max_bit_rate
         try:
             # Extract numeric part and unit (k or m)
             numeric_maxrate = int(re.sub(r'\D', '', max_bit_rate))
             unit = re.sub(r'\d', '', max_bit_rate).lower()
-            if unit not in ['k', 'm']: unit = 'k' # Default to k if unit is missing/invalid
+            if unit not in ['k', 'm']: 
+                unit = 'k'
             bufsize_val = numeric_maxrate * 2
             current_settings['bufsize'] = f"{bufsize_val}{unit}"
-            # Ensure rate control mode supports maxrate (e.g., vbr for NVENC, not constqp)
-            if (codec.endswith("_NVENC") or codec.endswith("_QSV")) and current_settings.get('rc') == 'constqp':
-                 current_settings['rc'] = 'vbr' # Switch to VBR if maxrate is set
-                 if logging_enabled:
-                     print("Switched rate control to 'vbr' due to max_bit_rate setting.")
+            
+            # Enable VBR when maxrate is provided for NVENC/QSV
+            if codec.endswith("_nvenc") or codec.endswith("_qsv"):
+                current_settings['rc'] = 'vbr'
+                if logging_enabled:
+                    print(f"Enabled VBR rate control for {codec} due to maxrate setting")
+            
             if logging_enabled:
                 print(f"Applying maxrate: {current_settings['maxrate']}, bufsize: {current_settings['bufsize']}")
-        except ValueError:
-            # Keep warning prints
-            print(f"Warning: Could not parse max_bit_rate '{max_bit_rate}'. Ignoring.")
-        except Exception as e:
-             # Keep warning prints
-             print(f"Warning: Error processing max_bit_rate '{max_bit_rate}': {e}. Ignoring.")
+        except (ValueError, Exception) as e:
+            print(f"Warning: Could not parse max_bit_rate '{max_bit_rate}': {e}. Ignoring.")
 
+    # 7. Handle CRF usage - disable constqp when CRF is used
+    try:
+        uses_crf = 'crf' in current_settings
+        has_maxrate = 'maxrate' in current_settings
+        
+        if uses_crf:
+            # Remove constqp explicitly when CRF is used
+            if str(current_settings.get('rc', '')).lower() == 'constqp':
+                if logging_enabled:
+                    print("Disabling 'constqp' because CRF is in use")
+                del current_settings['rc']
+            
+            # For NVENC/QSV with CRF + maxrate, use VBR
+            if (codec.endswith('_nvenc') or codec.endswith('_qsv')) and has_maxrate:
+                current_settings['rc'] = 'vbr'
+                if logging_enabled:
+                    print(f"Using 'vbr' with CRF due to specified maxrate")
+    except Exception:
+        pass
 
-    # --- Build output_args dictionary for ffmpeg-python ---
+    # 8. CQ policy for NVENC/QSV - use VBR with CQ unless maxrate + CQ requires constqp
+    try:
+        has_cq = 'cq' in current_settings and isinstance(current_settings.get('cq'), (int, float))
+        has_maxrate = 'maxrate' in current_settings
+        
+        if (codec.endswith('_nvenc') or codec.endswith('_qsv')) and has_cq:
+            if has_maxrate:
+                # Use VBR with both CQ and maxrate for rate-limited quality encoding
+                current_settings['rc'] = 'vbr'
+                if 'qp' in current_settings:
+                    del current_settings['qp']
+                if logging_enabled:
+                    print(f"Using VBR with CQ and maxrate for {codec}")
+            else:
+                # Standard CQ encoding with VBR
+                current_settings['rc'] = 'vbr'
+                if 'qp' in current_settings:
+                    del current_settings['qp']
+                if logging_enabled:
+                    print(f"Using VBR with CQ for {codec}")
+    except Exception:
+        pass
+
+    # --- Build FFmpeg arguments ---
     output_args = {}
     
-    # Map internal setting names to actual ffmpeg argument names
+    # Map internal setting names to FFmpeg argument names
     key_map = {
-        'keyint': 'g',   
+        'keyint': 'g',
         'bitrate': 'b:v',
         'codec': 'vcodec'
     }
 
-    # Add all current settings to output_args, applying key mapping
+    # For NVENC in constqp mode, translate 'cq' to 'qp'
+    try:
+        if codec.endswith('_nvenc') and str(current_settings.get('rc', '')).lower() == 'constqp':
+            if 'cq' in current_settings and 'qp' not in current_settings:
+                current_settings['qp'] = int(current_settings.pop('cq'))
+                if logging_enabled:
+                    print("Translating NVENC constqp: using -qp instead of -cq")
+    except Exception:
+        pass
+
+    # Add all current settings to output_args
     for key, value in current_settings.items():
-        ffmpeg_key = key_map.get(key, key) # Use mapped key or original key if not in map
+        ffmpeg_key = key_map.get(key, key)
         output_args[ffmpeg_key] = value
 
-    # Ensure vcodec is explicitly set if it wasn't the primary 'codec' key
+    # Ensure vcodec is set
     if 'vcodec' not in output_args and 'codec' in current_settings:
-         output_args['vcodec'] = current_settings['codec']
+        output_args['vcodec'] = current_settings['codec']
 
-    # Remove the internal 'codec' key if it's different from 'vcodec' and still present
+    # Remove internal 'codec' key if different from 'vcodec'
     if 'codec' in output_args and output_args.get('codec') != output_args.get('vcodec'):
-         del output_args['codec']
+        del output_args['codec']
 
-    # **AUDIO HANDLING - Check if audio exists first**
+    # --- Audio handling ---
     has_audio = check_audio_stream(input_path)
     
     if has_audio:
         output_args['acodec'] = 'copy'
-        # Map video and audio streams separately - don't use a list
-        output_args['map'] = '0'  # Map all streams, let FFmpeg handle it
+        output_args['map'] = '0'  # Map all streams
     else:
-        # No audio - only map video
         output_args['map'] = '0:v:0'  # Only video stream
     
     if logging_enabled:
@@ -320,9 +409,7 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
     # --- Execute FFmpeg ---
     try:
         start_time = time.time()
-        log_level = 'info' if logging_enabled else 'error'
         
-        # Create input and output with proper stream mapping
         input_stream = ffmpeg.input(input_path)
         output_stream = ffmpeg.output(input_stream, output_path, **output_args)
         
@@ -337,11 +424,11 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
         encoding_time_calculated = round(end_time - start_time, 2)
         stderr = result[1].decode("utf-8") if result[1] else ""
 
-        # Extract final encoding log line (optional, for debugging)
+        # Extract final encoding log line for debugging
         encoding_results_log = None
-        if logging_enabled: # Only parse log if logging is on
+        if logging_enabled:
             for line in stderr.splitlines():
-                if "time=" in line: # Find the last line reporting time/stats
+                if "time=" in line:
                     encoding_results_log = line
 
         if logging_enabled:
@@ -349,14 +436,50 @@ def encode_video(input_path, output_path, codec, rate=None, max_bit_rate=None, p
         return encoding_results_log, encoding_time_calculated
 
     except ffmpeg.Error as e:
-        print(f"Error encoding with {codec} scene '{scene_type}': {e}")
-        print(f"FFmpeg stderr: {e.stderr.decode('utf8')}")
-        return None, None
-    except Exception as e:
-        print(f"Unexpected error during encoding with {codec} scene '{scene_type}': {e}")
-        traceback.print_exc()
-        return None, None
-    
+        err_text = e.stderr.decode('utf8') if getattr(e, 'stderr', None) else str(e)
+        if logging_enabled:
+            print(f"Error encoding with {codec} scene '{scene_type}': {e}")
+            print(f"FFmpeg stderr: {err_text}")
+            print("Trying with libsvtav1 encoder...")
+
+        try:
+            # switch codec from gpu to cpu
+            output_args["vcodec"] = "libsvtav1"
+
+            # remove nvenc-specific options that might cause fallback errors
+            for key in ["spatial-aq", "temporal-aq", "aq-strength", "rc-lookahead", "preset"]:
+                output_args.pop(key, None)
+
+            start_time = time.time()
+            input_stream = ffmpeg.input(input_path)
+            output_stream = ffmpeg.output(input_stream, output_path, **output_args)
+
+            result = output_stream.run(
+                overwrite_output=True,
+                capture_stdout=True,
+                capture_stderr=True,
+                quiet=(not logging_enabled)
+            )
+
+            end_time = time.time()
+            encoding_time_calculated = round(end_time - start_time, 2)
+            stderr = result[1].decode("utf-8") if result[1] else ""
+
+            encoding_results_log = None
+            if logging_enabled:
+                for line in stderr.splitlines():
+                    if "time=" in line:
+                        encoding_results_log = line
+
+            if logging_enabled:
+                print(f"Successfully encoded using libsvtav1 fallback for scene '{scene_type}': {output_path}")
+            return encoding_results_log, encoding_time_calculated
+
+        except Exception as fallback_error:
+            if logging_enabled:
+                print(f"Fallback encoding with libsvtav1 also failed: {fallback_error}")
+            return None, None
+            
 def encode_lossless_video(input_path, output_path, logging_enabled=True):
     """
     Encode video using FFV1 lossless codec with optimized settings.

services/compress/utils/encoder_configs.py

@@ -1,72 +1,56 @@
 
+# Configurable parameters for all encoders
+# Base settings, including default AQ where applicable
 ENCODER_SETTINGS = {
-    "AV1_Optimized": {
-        "codec": "libsvtav1", "preset": "8", "crf": 30, "keyint": 50,
-        # Add SVT-AV1 specific AQ if needed, e.g., --enable-variance-boost
-    },
-    "libsvtav1": {
+    
+    "libsvtav1": {  # Changed from "AV1_Optimized"
         "codec": "libsvtav1", "preset": "8", "crf": 30, "keyint": 50,
-        # SVT-AV1 encoder settings
+        # SVT-AV1 specific settings
     },
     "av1_nvenc": {
-        "codec": "av1_nvenc", "preset": "p6", "cq": 30, "keyint": 50, 'pix_fmt': 'yuv420p' # Default AQ settings for NVENC AV1
+        "codec": "av1_nvenc", "preset": "p6", "cq": 30, "keyint": 50, 'pix_fmt': 'yuv420p'
     },
-    "av1_rust": {
-        "codec": "librav1e", "preset": "4", "crf": 35, "keyint": 50,
-        # rav1e has AQ settings too, check its documentation
+    "libvpx_vp9": {  # Changed from "vp9"
+        "codec": "libvpx-vp9", "deadline": "good", "cpu-used": 2, "crf": 32, "keyint": 50,
+        "aq-mode": 1, "arnr-maxframes": 7, "arnr-strength": 4, "auto-alt-ref": 1,
+        "tune": "psnr", "row-mt": 1
     },
-    "av1_fallback": {
-        "codec": "libaom-av1", "preset": "medium", "crf": 30, "b:v": "0", "cpu-used": 4, "keyint": 50,
-        "aq-mode": 1 # Default AQ mode for libaom
-    },
-    "h264": {
+    "libx264": {  # Changed from "h264"
         "codec": "libx264", "preset": "medium", "crf": 23, "keyint": 50,
-        "aq-mode": 1, "aq-strength": 1.0 # Default AQ for x264
+        "aq-mode": 1, "aq-strength": 1.0
     },
-    "h266_vvc": {
-        "codec": "libvvenc", "preset": "p4", "crf": 28, "keyint": 50,
-        # Check libvvenc documentation for AQ parameters
+    "libvvenc": {  # Changed from "h266_vvc"
+        "codec": "libvvenc", "preset": "medium", "crf": 28, "keyint": 50,
+        # VVC/H.266 encoder
     },
-    "hevc": {
+    "libx265": {  # Changed from "hevc"
         "codec": "libx265", "preset": "medium", "crf": 28, "keyint": 50,
-        "aq-mode": 2, "aq-strength": 1.0 # Default AQ for x265
+        "aq-mode": 2, "aq-strength": 1.0
     },
     "hevc_nvenc": {
         "codec": "hevc_nvenc", "preset": "p4", "rc": "constqp", "cq": 22, "keyint": 50,
-        "spatial-aq": 1, "temporal-aq": 0 # Default AQ settings for NVENC HEVC
+        "spatial-aq": 1, "temporal-aq": 0
     },
     "h264_nvenc": {
         "codec": "h264_nvenc", "preset": "p4", "rc": "constqp", "cq": 22, "keyint": 50,
-        "spatial-aq": 1, "temporal-aq": 0 # Default AQ settings for NVENC H264
-    },
-    "h264_videotoolbox": {
-        "codec": "h264_videotoolbox", 
-        "profile:v": "main", 
-        "crf": 23, 
-        "keyint": 50,
-        'pix_fmt': 'yuv420p' # Often required for hardware encoders
-    },
-    "hevc_videotoolbox": {
-        "codec": "hevc_videotoolbox",
-        "profile:v": "main", 
-        "crf": 28, 
-        "keyint": 50,
-        'pix_fmt': 'yuv420p'
+        "spatial-aq": 1, "temporal-aq": 0
     },
     "ffv1": {
         "codec": "ffv1",
-        "level": 3,           
-        "coder": 1,           
-        "context": 1,         
-        "g": 1,               
-        "slices": 4,          
-        "slicecrc": 1,        
-        "pix_fmt": "yuv420p", 
+        "level": 3,
+        "coder": 1,
+        "context": 1,
+        "g": 1,
+        "slices": 4,
+        "slicecrc": 1,
+        "pix_fmt": "yuv420p",
     },
 }
 
+# Scene-Specific Parameter Overrides (including AQ and keyint)
+# These are examples and need tuning based on content and codec specifics.
 SCENE_SPECIFIC_PARAMS = {
-    'AV1_NVENC': {
+    'av1_nvenc': {  # Changed from 'AV1_NVENC'
         'Screen Content / Text': {'preset': 'p7', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 250},
         'Faces / People': {'preset': 'p6', 'spatial-aq': 1, 'temporal-aq': 1, 'keyint': 100},
         'Animation / Cartoon / Rendered Graphics': {'preset': 'p5', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 150},
@@ -74,7 +58,7 @@ SCENE_SPECIFIC_PARAMS = {
         'other': {'keyint': 100},
         'unclear': {'keyint': 100},
     },
-    'HEVC_NVENC': {
+    'hevc_nvenc': {  # Changed from 'HEVC_NVENC'
         'Screen Content / Text': {'preset': 'p7', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 250},
         'Faces / People': {'preset': 'p6', 'spatial-aq': 1, 'temporal-aq': 1, 'keyint': 100},
         'Animation / Cartoon / Rendered Graphics': {'preset': 'p5', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 150},
@@ -82,7 +66,7 @@ SCENE_SPECIFIC_PARAMS = {
         'other': {'keyint': 100},
         'unclear': {'keyint': 100},
     },
-    'H264_NVENC': {
+    'h264_nvenc': {  # Changed from 'H264_NVENC'
         'Screen Content / Text': {'preset': 'p7', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 250},
         'Faces / People': {'preset': 'p6', 'spatial-aq': 1, 'temporal-aq': 1, 'keyint': 100},
         'Animation / Cartoon / Rendered Graphics': {'preset': 'p5', 'spatial-aq': 1, 'temporal-aq': 0, 'keyint': 150},
@@ -90,14 +74,6 @@ SCENE_SPECIFIC_PARAMS = {
         'other': {'keyint': 100},
         'unclear': {'keyint': 100},
     },
-    'libaom-av1': {
-        'Screen Content / Text': {'cpu-used': 3, 'tune': 'ssim', 'aq-mode': 2, 'keyint': 250},
-        'Animation / Cartoon / Rendered Graphics': {'cpu-used': 5, 'tune': 'animation', 'aq-mode': 1, 'keyint': 150},
-        'Faces / People': {'cpu-used': 4, 'tune': 'psnr', 'aq-mode': 1, 'keyint': 100},
-        'Gaming Content': {'cpu-used': 6, 'tune': 'fastdecode', 'aq-mode': 1, 'keyint': 75},
-        'other': {'cpu-used': 4, 'tune': 'film', 'aq-mode': 1, 'keyint': 100},
-        'unclear': {'cpu-used': 4, 'tune': 'film', 'aq-mode': 1, 'keyint': 100},
-    },
     'libx264': {
         'Screen Content / Text': {'preset': 'slow', 'tune': 'film', 'aq-mode': 1, 'aq-strength': 1.2, 'keyint': 250},
         'Animation / Cartoon / Rendered Graphics': {'preset': 'medium', 'tune': 'animation', 'aq-mode': 1, 'aq-strength': 1.0, 'keyint': 150},
@@ -106,86 +82,163 @@ SCENE_SPECIFIC_PARAMS = {
         'other': {'preset': 'medium', 'tune': 'film', 'aq-mode': 1, 'aq-strength': 1.0, 'keyint': 100},
         'unclear': {'preset': 'medium', 'tune': 'film', 'aq-mode': 1, 'aq-strength': 1.0, 'keyint': 100},
     },
-     'libx265': {
+    'libx265': {
         'Screen Content / Text': {'preset': 'medium', 'tune': 'grain', 'aq-mode': 2, 'aq-strength': 1.1, 'keyint': 250},
         'Animation / Cartoon / Rendered Graphics': {'preset': 'medium', 'tune': 'animation', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 150},
-        'Faces / People': {'preset': 'medium', 'tune': 'none', 'aq-mode': 2, 'aq-strength': 0.9, 'keyint': 100},
+        'Faces / People': {'preset': 'medium', 'aq-mode': 2, 'aq-strength': 0.9, 'keyint': 100},
         'Gaming Content': {'preset': 'fast', 'tune': 'fastdecode', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 75},
-        'other': {'preset': 'medium', 'tune': 'none', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 100},
-        'unclear': {'preset': 'medium', 'tune': 'none', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 100},
+        'other': {'preset': 'medium', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 100},
+        'unclear': {'preset': 'medium', 'aq-mode': 2, 'aq-strength': 1.0, 'keyint': 100},
+    },
+    'libvpx_vp9': {
+        'Screen Content / Text': {'deadline': 'best', 'cpu-used': 1, 'aq-mode': 2, 'keyint': 250},
+        'Faces / People': {'deadline': 'good', 'cpu-used': 2, 'aq-mode': 1, 'keyint': 100},
+        'Animation / Cartoon / Rendered Graphics': {'deadline': 'good', 'cpu-used': 3, 'aq-mode': 1, 'keyint': 150},
+        'Gaming Content': {'deadline': 'realtime', 'cpu-used': 5, 'aq-mode': 1, 'keyint': 75},
+        'other': {'deadline': 'good', 'cpu-used': 2, 'aq-mode': 1, 'keyint': 100},
+        'unclear': {'deadline': 'good', 'cpu-used': 2, 'aq-mode': 1, 'keyint': 100},
+    },
+    'libsvtav1': {
+        'Screen Content / Text': {'preset': '6', 'tune': 0, 'keyint': 250},  # tune 0 = subjective quality
+        'Faces / People': {'preset': '8', 'tune': 1, 'keyint': 100},        # tune 1 = objective quality
+        'Animation / Cartoon / Rendered Graphics': {'preset': '7', 'tune': 0, 'keyint': 150},
+        'Gaming Content': {'preset': '9', 'tune': 2, 'keyint': 75},         # tune 2 = fast decode
+        'other': {'preset': '8', 'tune': 1, 'keyint': 100},
+        'unclear': {'preset': '8', 'tune': 1, 'keyint': 100},
+    },
+    'libvvenc': {
+        'Screen Content / Text': {'preset': 'slow', 'keyint': 250},
+        'Faces / People': {'preset': 'medium', 'keyint': 100},
+        'Animation / Cartoon / Rendered Graphics': {'preset': 'medium', 'keyint': 150},
+        'Gaming Content': {'preset': 'fast', 'keyint': 75},
+        'other': {'preset': 'medium', 'keyint': 100},
+        'unclear': {'preset': 'medium', 'keyint': 100},
     },
 }
 
-MODEL_CQ_REFERENCE_CODEC = "libsvtav1" 
 
+
+# --- START: New Configuration for Quality Parameter Mapping ---
+
+# Define which codec's CQ scale your VMAF prediction model was trained on.
+# Example: If your model predicts CQ values suitable for AV1_NVENC.
+MODEL_CQ_REFERENCE_CODEC = "av1_nvenc" # IMPORTANT: User must set this correctly
+
+# Define anchor points for mapping the model's reference CQ (from MODEL_CQ_REFERENCE_CODEC)
+# to other codecs' quality parameters.
+# This requires empirical tuning and knowledge of codec quality scales.
+# 'model_ref_cq_range': The typical input CQ range from your VMAF model.
+# 'target_param_type': 'cq' or 'crf' for the target codec.
+# 'target_param_range': The typical/effective output range for the target codec's parameter.
+# 'anchor_points': A list of [model_ref_cq_value, target_codec_param_value] pairs.
+#                  These points define the mapping relationship.
 QUALITY_MAPPING_ANCHORS = {
-    "libaom-av1": { 
-        "model_ref_cq_range": [10, 63], 
+    #   Changed "libaom-av1" to match ENCODER_SETTINGS
+    #   Changed "H264" to "libx264" to match ENCODER_SETTINGS
+    "libx264": { # Mapping av1_nvenc CQ to libx264 CRF
+        "model_ref_cq_range": [10, 63],
         "target_param_type": "crf",
-        "target_param_range": [10, 63], 
-        "anchor_points": [ 
-            [20, 22],
-            [30, 32],
-            [40, 42],
-            [50, 52]
+        "target_param_range": [0, 51],  # Typical CRF range for libx264
+        "anchor_points": [ # [model_av1_nvenc_cq, libx264_crf]
+            [20, 16], #   H.264 is less efficient, needs lower CRF
+            [30, 21], #   More realistic mapping
+            [40, 26], #   Conservative mapping for quality
+            [50, 31]  #   Upper range adjustment
         ]
     },
-    "H264": { 
+    #   Changed "HEVC_NVENC" to "hevc_nvenc" to match ENCODER_SETTINGS
+    "hevc_nvenc": { # Mapping av1_nvenc CQ to hevc_nvenc CQ
+        "model_ref_cq_range": [10, 63],
+        "target_param_type": "cq",
+        "target_param_range": [10, 51], # Typical CQ range for NVENC HEVC
+        "anchor_points": [ # [model_av1_nvenc_cq, hevc_nvenc_cq]
+            [20, 18], #  HEVC NVENC slightly different scale
+            [30, 23], #  More conservative mapping
+            [40, 28], #  Quality preservation
+            [50, 33]  #  Upper range adjustment
+        ]
+    },
+    "libx265": { # Mapping av1_nvenc CQ to libx265 CRF
         "model_ref_cq_range": [10, 63],
         "target_param_type": "crf",
-        "target_param_range": [0, 51], 
-        "anchor_points": [ 
-            [20, 18],
-            [30, 23],
-            [40, 28],
-            [50, 33]
+        "target_param_range": [0, 51], # Typical CRF range for libx265
+        "anchor_points": [ # [model_av1_nvenc_cq, libx265_crf]
+            [20, 18], #  x265 is more efficient than H.264
+            [30, 23], #  Similar to H.264 but slightly better
+            [40, 28], #  Conservative quality mapping
+            [50, 33]  #  Upper range adjustment
         ]
     },
-
-    "HEVC_NVENC": { 
+    #   Missing codec mappings
+    "h264_nvenc": { # Mapping av1_nvenc CQ to h264_nvenc CQ
         "model_ref_cq_range": [10, 63],
         "target_param_type": "cq",
-        "target_param_range": [10, 51], 
-        "anchor_points": [ 
-            [20, 19], 
-            [30, 25],
-            [40, 31],
-            [50, 37]
+        "target_param_range": [10, 51], # Typical CQ range for NVENC H.264
+        "anchor_points": [ # [model_av1_nvenc_cq, h264_nvenc_cq]
+            [20, 17], # NVENC H.264 needs lower CQ for similar quality
+            [30, 22],
+            [40, 27],
+            [50, 32]
+        ]
+    },
+    "libvpx_vp9": { # Mapping av1_nvenc CQ to libvpx-vp9 CRF
+        "model_ref_cq_range": [10, 63],
+        "target_param_type": "crf",
+        "target_param_range": [0, 63], # VP9 CRF range
+        "anchor_points": [ # [model_av1_nvenc_cq, vp9_crf]
+            [20, 22], # VP9 similar efficiency to AV1
+            [30, 32],
+            [40, 42],
+            [50, 52]
         ]
     },
-    "libx265": { 
+    "libsvtav1": { # Mapping av1_nvenc CQ to libsvtav1 CRF
         "model_ref_cq_range": [10, 63],
         "target_param_type": "crf",
-        "target_param_range": [0, 51], 
-        "anchor_points": [ 
-            [20, 20],
-            [30, 26],
-            [40, 32],
-            [50, 38]
+        "target_param_range": [0, 63], # SVT-AV1 CRF range
+        "anchor_points": [ # [model_av1_nvenc_cq, svt_av1_crf]
+            [20, 21], # SVT-AV1 slightly different from NVENC
+            [30, 31],
+            [40, 41],
+            [50, 51]
         ]
     },
+    "libvvenc": { # Mapping av1_nvenc CQ to libvvenc CRF
+        "model_ref_cq_range": [10, 63],
+        "target_param_type": "crf",
+        "target_param_range": [0, 51], # H.266/VVC CRF range
+        "anchor_points": [ # [model_av1_nvenc_cq, vvc_crf]
+            [20, 20], # VVC should be more efficient than AV1
+            [30, 25], # Better compression than AV1
+            [40, 30],
+            [50, 35]
+        ]
+    },
+    # If a codec is the same as MODEL_CQ_REFERENCE_CODEC, it will use the CQ directly.
 }
 
+# =============================================================================
+# CODEC-SPECIFIC CQ LIMITS AND QUALITY MAPPINGS
+# =============================================================================
+
+# Define maximum recommended CQ values for different codecs to maintain reasonable quality
 CODEC_CQ_LIMITS = {
+    # AV1 Codecs
     'av1_nvenc': {
-        'max_cq': 45,          
-        'recommended_max': 40,  
-        'quality_range': (20, 40), 
+        'max_cq': 45,           # Maximum CQ for reasonable quality
+        'recommended_max': 40,   # Conservative maximum for good quality
+        'quality_range': (20, 40),  # Typical quality range
         'description': 'NVIDIA AV1 encoder'
     },
-    'libsvtav1': {
+    'libsvtav1': {  #   Standardized name
         'max_cq': 50,
         'recommended_max': 45,
         'quality_range': (25, 45),
         'description': 'SVT-AV1 encoder'
     },
-    'libaom-av1': {
-        'max_cq': 50,
-        'recommended_max': 45,
-        'quality_range': (25, 45),
-        'description': 'AOM AV1 encoder'
-    },
     
+    
+    # H.264 Codecs
     'h264_nvenc': {
         'max_cq': 32,
         'recommended_max': 28,
@@ -199,6 +252,7 @@ CODEC_CQ_LIMITS = {
         'description': 'x264 H.264 encoder'
     },
     
+    # H.265/HEVC Codecs
     'hevc_nvenc': {
         'max_cq': 35,
         'recommended_max': 30,
@@ -212,13 +266,31 @@ CODEC_CQ_LIMITS = {
         'description': 'x265 HEVC encoder'
     },
     
-    'libvpx-vp9': {
+    # VP9 Codec
+    'libvpx_vp9': {  #   Changed from 'libvpx-vp9'
         'max_cq': 45,
         'recommended_max': 40,
         'quality_range': (25, 40),
         'description': 'VP9 encoder'
     },
     
+    #   H.266/VVC Codec
+    'libvvenc': {
+        'max_cq': 45,
+        'recommended_max': 40,
+        'quality_range': (20, 40),
+        'description': 'VVenC H.266 encoder'
+    },
+    
+    #   Lossless codec
+    'ffv1': {
+        'max_cq': 0,     # Lossless doesn't use quality parameters
+        'recommended_max': 0,
+        'quality_range': (0, 0),
+        'description': 'FFV1 lossless encoder'
+    },
+    
+    # Default fallback for unknown codecs
     'default': {
         'max_cq': 40,
         'recommended_max': 35,

services/compress/utils/merge_videos.py

@@ -19,35 +19,97 @@ def has_audio(video_path):
 def merge_videos(scene_videos, output_video, logging_enabled=True):
     """
     Merge the list of re-encoded scene files using FFmpeg's concat filter.
-    This function checks whether the scene files have audio or not, and builds the
-    concat filter accordingly.
+    Uses precise frame-level concatenation to prevent stuttering.
     
     Args:
         scene_files (list): List of file paths to the scene videos.
         output_video (str): Path to the final merged video.
         logging_enabled (bool): Whether to print messages.
     """
+    if not scene_videos:
+        if logging_enabled:
+            print("❌ No scene videos to merge")
+        return False
+    
+    # For single scene, just copy the file
+    if len(scene_videos) == 1:
+        try:
+            import shutil
+            shutil.copy2(scene_videos[0], output_video)
+            if logging_enabled:
+                print(f"✅ Single scene copied to {output_video}")
+            return True
+        except Exception as e:
+            if logging_enabled:
+                print(f"❌ Failed to copy single scene: {e}")
+            return False
+    
     # Create a temporary file with the correct format
     with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
         for video in scene_videos:
+            # Verify each scene file exists and has content
+            if not os.path.exists(video):
+                if logging_enabled:
+                    print(f"⚠️ Scene file missing: {video}")
+                return False
+            if os.path.getsize(video) == 0:
+                if logging_enabled:
+                    print(f"⚠️ Scene file empty: {video}")
+                return False
             # Write each file on a separate line with proper syntax
             f.write(f"file '{os.path.abspath(video)}'\n")
         concat_file = f.name
     
-    # Use the concat demuxer with the file list
+    # Use the concat demuxer with improved frame handling
     try:
+        # Use concat filter instead of demuxer for better frame alignment
         cmd = [
-            'ffmpeg', '-y', '-f', 'concat', '-safe', '0', 
-            '-i', concat_file, '-c', 'copy', output_video
+            'ffmpeg', '-y', '-hide_banner',
+            '-f', 'concat', '-safe', '0', 
+            '-i', concat_file,
+            '-c:v', 'copy',        # Video stream copy
+            '-c:a', 'copy',        # Audio stream copy  
+            '-avoid_negative_ts', 'make_zero',  # Handle timestamp alignment
+            '-fflags', '+genpts',  # Generate proper timestamps
+            output_video
         ]
-        subprocess.run(cmd, check=True, capture_output=True)
+        
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+        
+        if result.returncode == 0:
+            # Verify output file
+            if os.path.exists(output_video) and os.path.getsize(output_video) > 0:
+                if logging_enabled:
+                    print(f"✅ Successfully merged {len(scene_videos)} scenes into {output_video}")
+                return True
+            else:
+                if logging_enabled:
+                    print(f"❌ Merge failed: output file not created or empty")
+                return False
+        else:
+            if logging_enabled:
+                print(f"❌ FFmpeg merge failed (code {result.returncode})")
+                if result.stderr:
+                    print(f"Error details: {result.stderr}")
+            return False
+            
+    except subprocess.TimeoutExpired:
         if logging_enabled:
-            print(f"Successfully merged videos into {output_video}")
-        return True
+            print(f"❌ Merge process timed out after 5 minutes")
+        return False
     except subprocess.CalledProcessError as e:
         if logging_enabled:
-            print(f"Error merging videos: {e.stderr}")
+            print(f"❌ Error merging videos: {e}")
+            if hasattr(e, 'stderr') and e.stderr:
+                print(f"Error details: {e.stderr}")
+        return False
+    except Exception as e:
+        if logging_enabled:
+            print(f"❌ Unexpected error during merge: {e}")
         return False
     finally:
         # Clean up temp file
-        os.unlink(concat_file)
+        try:
+            os.unlink(concat_file)
+        except:
+            pass

services/compress/utils/processing_utils.py

@@ -5,6 +5,7 @@ import json
 import shutil
 from .encode_video import encode_video
 from .classify_scene import classify_scene_with_model, extract_frames_from_scene
+from .analyze_video_fast import analyze_video_fast
 
 def get_video_bitrate(video_path):
     """Get the bitrate of a video file in kbps."""
@@ -403,7 +404,7 @@ def classify_scene_from_path(scene_path, temp_dir, scene_classifier_model, avail
             print(f"Extracting video features from {scene_path}")
         
         try:
-            from video_analysis import analyze_video_fast
+            
             video_features = analyze_video_fast(scene_path, logging_enabled=False)
             
             if not video_features or 'error' in video_features:

services/compress/utils/video_utils.py

@@ -49,3 +49,40 @@ def get_video_codec(video_path):
     except Exception as e:
         print(f"An unexpected error occurred while getting video codec: {e}")
         return None
+
+def get_keyframes(video_path: str, max_keyframes: int = 50):
+    """Get keyframe positions in seconds."""
+    try:
+        # Use ffprobe to get keyframe timestamps
+        cmd = [
+            'ffprobe', '-v', 'error',
+            '-select_streams', 'v:0',
+            '-show_entries', 'packet=pts_time,flags',
+            '-of', 'csv=print_section=0',
+            video_path
+        ]
+        
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        
+        if result.returncode != 0:
+            print(f"⚠️ Failed to get keyframes: ffprobe returned {result.returncode}")
+            return []
+        
+        # Parse output to find keyframe timestamps
+        keyframes = []
+        for line in result.stdout.splitlines():
+            if "K_" in line or ",K" in line:  # Keyframe flag
+                parts = line.split(',')
+                if len(parts) >= 1 and parts[0].replace('.', '', 1).isdigit():
+                    timestamp = float(parts[0])
+                    keyframes.append(timestamp)
+                    
+                    # Limit to max_keyframes to prevent excessive processing
+                    if len(keyframes) >= max_keyframes:
+                        break
+        
+        return keyframes
+        
+    except Exception as e:
+        print(f"❌ Error getting keyframes: {str(e)}")
+        return []

services/compress/validator_merger.py

@@ -165,8 +165,6 @@ def validation_and_merging(original_video_path, encoded_scenes_data, config, log
         print(f"      🎬 Executing video merge...")
     
     try:
-        
-        
         def timeout_handler(signum, frame):
             raise TimeoutError("Video merging timeout")
         

services/compress/video_preprocessor.py

@@ -31,7 +31,7 @@ def pre_processing(video_path, target_quality='Medium',codec='auto', max_duratio
     
     # ✅ CODEC HANDLING: Resolve auto codec selection
     if codec.lower() == 'auto':
-        target_codec = 'libsvtav1'  # Default to SVT-AV1 for best quality
+        target_codec = 'av1_nvenc'  # Default to AV1 for best quality
         
     else:
         target_codec = codec
@@ -40,8 +40,8 @@ def pre_processing(video_path, target_quality='Medium',codec='auto', max_duratio
     # ✅ QUALITY MAPPING: Convert target quality to VMAF score
     quality_vmaf_mapping = {
         'High': 95.0,
-        'Medium': 93.0,
-        'Low': 90.0
+        'Medium': 90.0,  # Updated from 93.0 to 90.0
+        'Low': 85.0      # Updated from 90.0 to 85.0
     }
     
     # Validate and convert target quality
@@ -158,3 +158,5 @@ def pre_processing(video_path, target_quality='Medium',codec='auto', max_duratio
                 'container': os.path.splitext(video_path)[1][1:]  # Extension without dot
             }
         }
+
+