Add automatic document detection and collection clearing: Implement file hash tracking to automatically clear all Qdrant collections when a new document is uploaded, ensuring data isolation between different documents. Added manual reset button in Data Management tab for force clearing when reprocessing same file."

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 import logging
 import os
+import hashlib
 from typing import List, Tuple, Optional, Dict, Any
 import pandas as pd
 from qdrant_client import QdrantClient
@@ -24,6 +25,9 @@ logger = logging.getLogger(__name__)
 document_processor = None
 embeddings_pipeline = None
 
+# Global variable to track current document identity
+current_document_hash = None
+
 # OpenAI client (lazy init)
 _openai_client: Optional[OpenAI] = None
 
@@ -34,6 +38,58 @@ def _get_openai_client() -> OpenAI:
         _openai_client = OpenAI(api_key=api_key)
     return _openai_client
 
+def get_file_hash(file_path: str) -> str:
+    """
+    Calculate MD5 hash of a file to track document identity.
+    
+    Args:
+        file_path: Path to the file
+        
+    Returns:
+        MD5 hash string of the file content
+    """
+    hash_md5 = hashlib.md5()
+    try:
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+        return hash_md5.hexdigest()
+    except Exception as e:
+        logger.error(f"Error calculating file hash for {file_path}: {str(e)}")
+        return str(hash(file_path))  # Fallback to simple hash of path
+
+def clear_all_collections_for_new_document() -> bool:
+    """
+    Clear all collections in Qdrant when a new document is detected.
+    
+    Returns:
+        bool: True if clearing was successful or not needed, False on error
+    """
+    try:
+        logger.info("🗑️ Nuevo documento detectado: limpiando todas las colecciones existentes...")
+        result = delete_all_collections()
+        if "✅" in result:
+            logger.info("✅ Todas las colecciones fueron eliminadas exitosamente")
+            return True
+        else:
+            logger.warning(f"⚠️ Limpieza de colecciones completada con advertencias: {result}")
+            return True
+    except Exception as e:
+        logger.error(f"❌ Error al limpiar colecciones para nuevo documento: {str(e)}")
+        return False
+
+def reset_document_tracking() -> str:
+    """
+    Reset the document tracking hash to force collection clearing on next upload.
+    
+    Returns:
+        Status message
+    """
+    global current_document_hash
+    current_document_hash = None
+    logger.info("🔄 Document tracking reset - next upload will clear all collections")
+    return "✅ Document tracking reset. Next document upload will clear all existing collections."
+
 def initialize_components():
     """Initialize the pipeline components with error handling for cloud deployment."""
     global document_processor, embeddings_pipeline
@@ -57,6 +113,7 @@ def initialize_components():
 def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str, pd.DataFrame]:
     """
     Process a document with selected chunking methods.
+    Automatically clears all collections if a different document is detected.
     
     Args:
         file_path: Path to the uploaded file
@@ -65,6 +122,8 @@ def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str,
     Returns:
         Tuple of (status_message, chunks_dataframe)
     """
+    global current_document_hash
+    
     try:
         if document_processor is None:
             return "❌ Error: Document processor not initialized. Check system configuration.", pd.DataFrame()
@@ -78,6 +137,26 @@ def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str,
         if not chunking_methods:
             return "❌ Error: Please select at least one chunking method", pd.DataFrame()
         
+        # Calculate hash of the current document
+        new_document_hash = get_file_hash(file_path)
+        logger.info(f"Document hash: {new_document_hash}")
+        
+        # Check if this is a different document than the previous one
+        if current_document_hash is not None and current_document_hash != new_document_hash:
+            logger.info(f"🔄 Documento diferente detectado (hash anterior: {current_document_hash[:8]}..., nuevo: {new_document_hash[:8]}...)")
+            
+            # Clear all collections for the new document
+            clear_success = clear_all_collections_for_new_document()
+            if not clear_success:
+                logger.warning("⚠️ Falló la limpieza de colecciones, pero continuando con el procesamiento...")
+        elif current_document_hash is None:
+            logger.info("📄 Primer documento cargado en esta sesión")
+        else:
+            logger.info("📄 Mismo documento que el anterior, manteniendo embeddings existentes")
+        
+        # Update the current document hash
+        current_document_hash = new_document_hash
+        
         # Process the document
         logger.info(f"Processing document: {file_path}")
         chunks_by_method = document_processor.process_document(
@@ -117,6 +196,10 @@ def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str,
     except Exception as e:
         logger.error(f"Error processing document: {str(e)}")
         return f"❌ Error: {str(e)}", pd.DataFrame()
+        
+    except Exception as e:
+        logger.error(f"Error processing document: {str(e)}")
+        return f"❌ Error: {str(e)}", pd.DataFrame()
 
 def search_chunks(query: str, top_k: int = None) -> Tuple[str, pd.DataFrame]:
     """
@@ -913,6 +996,12 @@ with gr.Blocks(title="Document Chunking & Search System", theme=gr.themes.Soft()
                 gr.Markdown("**Delete All Collections:**")
                 gr.Markdown("⚠️ **Warning**: This will delete ALL collections permanently!")
                 delete_all_btn = gr.Button("🗑️ Delete All Collections", variant="stop")
+
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("**Reset Document Tracking:**")
+                gr.Markdown("🔄 Force collection clearing on next document upload (useful when reprocessing the same file)")
+                reset_tracking_btn = gr.Button("🔄 Reset Document Tracking", variant="secondary")
         
         delete_output = gr.Textbox(
             label="Deletion Status",
@@ -950,6 +1039,11 @@ with gr.Blocks(title="Document Chunking & Search System", theme=gr.themes.Soft()
             delete_all_collections,
             outputs=delete_output
         )
+        
+        reset_tracking_btn.click(
+            reset_document_tracking,
+            outputs=delete_output
+        )
 
 if __name__ == "__main__":
     # Set up environment for deployment