Spaces:

FLAFUENTE
/

optchunkopt

Sleeping

🚀 Fix HF Spaces timeout issues with Qdrant Cloud
d9c7d47 12 months ago
44.1 kB
import gradio as gr
import logging
import os
import hashlib
from typing import List, Tuple, Optional, Dict, Any
import pandas as pd
from qdrant_client import QdrantClient
import re
import json
from collections import defaultdict, Counter
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from openai import OpenAI

# Import your existing modules
from document_processor import DocumentProcessor
from embeddings_qdrant import EmbeddingsQdrantPipeline
import settings

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Global variables for components (will be initialized in main)
document_processor = None
embeddings_pipeline = None

# Global variable to track current document identity
current_document_hash = None

# OpenAI client (lazy init)
_openai_client: Optional[OpenAI] = None

def _get_openai_client() -> OpenAI:
    global _openai_client
    if _openai_client is None:
        api_key = getattr(settings, "OPENAI_API_KEY", os.getenv("OPENAI_API_KEY"))
        _openai_client = OpenAI(api_key=api_key)
    return _openai_client

def get_file_hash(file_path: str) -> str:
    """
    Calculate MD5 hash of a file to track document identity.
    
    Args:
        file_path: Path to the file
        
    Returns:
        MD5 hash string of the file content
    """
    hash_md5 = hashlib.md5()
    try:
        with open(file_path, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()
    except Exception as e:
        logger.error(f"Error calculating file hash for {file_path}: {str(e)}")
        return str(hash(file_path))  # Fallback to simple hash of path

def clear_all_collections_for_new_document() -> bool:
    """
    Clear all collections in Qdrant when a new document is detected.
    
    Returns:
        bool: True if clearing was successful or not needed, False on error
    """
    try:
        logger.info("🗑️ Nuevo documento detectado: limpiando todas las colecciones existentes...")
        result = delete_all_collections()
        if "✅" in result:
            logger.info("✅ Todas las colecciones fueron eliminadas exitosamente")
            return True
        else:
            logger.warning(f"⚠️ Limpieza de colecciones completada con advertencias: {result}")
            return True
    except Exception as e:
        logger.error(f"❌ Error al limpiar colecciones para nuevo documento: {str(e)}")
        return False

def reset_document_tracking() -> str:
    """
    Reset the document tracking hash to force collection clearing on next upload.
    
    Returns:
        Status message
    """
    global current_document_hash
    current_document_hash = None
    logger.info("🔄 Document tracking reset - next upload will clear all collections")
    return "✅ Seguimiento de documentos reiniciado. La próxima carga de documento eliminará todas las colecciones existentes."

def initialize_components():
    """Initialize the pipeline components with error handling for cloud deployment."""
    global document_processor, embeddings_pipeline
    
    try:
        document_processor = DocumentProcessor()
        logger.info("✅ Document processor initialized")
    except Exception as e:
        logger.error(f"❌ Failed to initialize document processor: {str(e)}")
        document_processor = None
    
    try:
        embeddings_pipeline = EmbeddingsQdrantPipeline()
        logger.info("✅ Embeddings pipeline initialized")
    except Exception as e:
        logger.error(f"❌ Failed to initialize embeddings pipeline: {str(e)}")
        embeddings_pipeline = None
        
    return document_processor is not None and embeddings_pipeline is not None

def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str, pd.DataFrame]:
    """
    Process a document with selected chunking methods.
    Automatically clears all collections if a different document is detected.
    
    Args:
        file_path: Path to the uploaded file
        chunking_methods: List of selected chunking methods
    
    Returns:
        Tuple of (status_message, chunks_dataframe)
    """
    global current_document_hash
    
    try:
        if document_processor is None:
            return "❌ Error: Procesador de documentos no inicializado. Verifica la configuración del sistema.", pd.DataFrame()
            
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado. Verifica la conexión a Qdrant y la clave API de OpenAI.", pd.DataFrame()
            
        if not file_path:
            return "❌ Error: No se ha cargado ningún archivo", pd.DataFrame()
        
        if not chunking_methods:
            return "❌ Error: Por favor selecciona al menos un método de chunking", pd.DataFrame()
        
        # Calculate hash of the current document
        new_document_hash = get_file_hash(file_path)
        logger.info(f"Document hash: {new_document_hash}")
        
        # Check if this is a different document than the previous one
        if current_document_hash is not None and current_document_hash != new_document_hash:
            logger.info(f"🔄 Documento diferente detectado (hash anterior: {current_document_hash[:8]}..., nuevo: {new_document_hash[:8]}...)")
            
            # Clear all collections for the new document
            clear_success = clear_all_collections_for_new_document()
            if not clear_success:
                logger.warning("⚠️ Falló la limpieza de colecciones, pero continuando con el procesamiento...")
        elif current_document_hash is None:
            logger.info("📄 Primer documento cargado en esta sesión")
        else:
            logger.info("📄 Mismo documento que el anterior, manteniendo embeddings existentes")
        
        # Update the current document hash
        current_document_hash = new_document_hash
        
        # Process the document
        logger.info(f"Processing document: {file_path}")
        chunks_by_method = document_processor.process_document(
            file_path=file_path,
            chunking_methods=chunking_methods
        )
        
        if not chunks_by_method:
            return "❌ Error: No se generaron chunks del documento", pd.DataFrame()
        
        # Store embeddings in Qdrant
        success = embeddings_pipeline.process_and_store_chunks(chunks_by_method)
        
        if not success:
            return "❌ Error: Falló el almacenamiento de embeddings en Qdrant", pd.DataFrame()
        
        # Create a summary dataframe
        summary_data = []
        total_chunks = 0
        for method, chunks in chunks_by_method.items():
            # Use a shorter sample chunk for better display
            sample_text = chunks[0][:150] + "..." if chunks and len(chunks[0]) > 150 else (chunks[0] if chunks else "Sin chunks")
            summary_data.append({
                "Método de Chunking": method.capitalize(),
                "Número de Chunks": len(chunks),
                "Longitud Promedio de Chunk": round(sum(len(chunk) for chunk in chunks) / len(chunks)) if chunks else 0,
                "Chunk de Muestra": sample_text
            })
            total_chunks += len(chunks)
        
        summary_df = pd.DataFrame(summary_data)
        
        status_msg = f"✅ ¡Éxito! Documento procesado con {len(chunking_methods)} métodos. Total de chunks: {total_chunks}"
        
        return status_msg, summary_df
        
    except Exception as e:
        logger.error(f"Error processing document: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()
        
    except Exception as e:
        logger.error(f"Error processing document: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

def search_chunks(query: str, top_k: int = None) -> Tuple[str, pd.DataFrame]:
    """
    Search for similar chunks across all collections.
    
    Args:
        query: Search query
        top_k: Number of results to return (uses settings default if None)
    
    Returns:
        Tuple of (status_message, results_dataframe)
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado. Verifica la conexión a Qdrant y la clave API de OpenAI.", pd.DataFrame()
            
        if not query.strip():
            return "❌ Error: Por favor ingresa una consulta de búsqueda", pd.DataFrame()
        
        # Use provided top_k or default from settings
        k = top_k if top_k is not None else settings.TOP_K_RESULTS
        
        # Perform search
        logger.info(f"Searching for: '{query}' (top_k={k})")
        results = embeddings_pipeline.search_chunks(query=query, top_k=k)
        
        if not results:
            return "⚠️ No se encontraron resultados. Prueba una consulta diferente o verifica si los documentos están procesados.", pd.DataFrame()
        
        # Create results dataframe
        results_data = []
        for i, (text, score, method) in enumerate(results, 1):
            # Better text preview with line breaks preserved
            text_preview = text[:300] + "..." if len(text) > 300 else text
            # Clean up text preview for better display
            text_preview = text_preview.replace('\n', ' ').replace('\r', ' ').strip()
            
            results_data.append({
                "Rango": i,
                "Puntuación de Similitud": round(score, 4),
                "Método de Chunking": method.capitalize(),
                "Vista Previa de Texto": text_preview,
                "Texto Completo": text
            })
        
        results_df = pd.DataFrame(results_data)
        
        status_msg = f"✅ Se encontraron {len(results)} resultados para la consulta: '{query}'"
        
        return status_msg, results_df
        
    except Exception as e:
        logger.error(f"Error searching chunks: {str(e)}")
        return f"❌ Error en la búsqueda: {str(e)}", pd.DataFrame()

def get_qdrant_dashboard_url() -> str:
    """
    Get the Qdrant dashboard URL with authentication for public access.
    
    Returns:
        Dashboard URL with access instructions
    """
    try:
        # For Qdrant Cloud, the dashboard is usually at the same URL
        base_url = settings.QDRANT_URL.replace(':6333', '').replace(':6334', '')
        dashboard_url = f"{base_url}/dashboard"
        
        return f"""🌐 **Acceso al Dashboard de Qdrant:**
**🔗 URL Directo:** {dashboard_url}
🔑 **Autenticación (si es necesaria):**
- Clave API: `{settings.QDRANT_API_KEY}`
- URL del Servidor: `{settings.QDRANT_URL}`
📋 **Cómo Acceder:**
1. Haz clic/copia la URL de arriba para abrir el dashboard de Qdrant
2. Si se solicita autenticación, usa la clave API proporcionada arriba
3. Explora tus colecciones, vectores y datos visualmente
⚡ **Lo que puedes hacer:**
- 📊 Ver estadísticas y metadatos de colecciones
- 🔍 Explorar chunks de documentos almacenados
- 📈 Visualizar vectores de embeddings y relaciones
- ⚙️ Monitorear rendimiento y salud del cluster
- 🔎 Buscar y filtrar tus datos interactivamente
💡 **Tip:** Bookmark this URL for quick access to your vector database!"""
        
    except Exception as e:
        logger.error(f"Error generating dashboard URL: {str(e)}")
        return f"❌ Error generating dashboard URL: {str(e)}"

def get_collections_summary() -> Tuple[str, pd.DataFrame]:
    """
    Get summary of all Qdrant collections using native APIs.
    
    Returns:
        Tuple of (status_message, collections_dataframe)
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado", pd.DataFrame()
        
        # Use Qdrant native API to get collections info
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication and extended timeout for HF Spaces
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(
                url=settings.QDRANT_URL, 
                api_key=settings.QDRANT_API_KEY,
                timeout=60.0,
                prefer_grpc=False
            )
        else:
            client = QdrantClient(
                url=settings.QDRANT_URL,
                timeout=60.0,
                prefer_grpc=False
            )
        
        # Get all collections
        collections = client.get_collections()
        
        collections_data = []
        total_chunks = 0
        
        for collection in collections.collections:
            try:
                # Get detailed info for each collection
                info = client.get_collection(collection.name)
                
                collections_data.append({
                    "Nombre de Colección": collection.name,
                    "Cantidad de Puntos": info.points_count,
                    "Cantidad de Vectores": info.vectors_count or info.points_count,
                    "Estado": info.status.value if hasattr(info.status, 'value') else str(info.status),
                    "Estado del Optimizador": "OK" if info.optimizer_status else "Optimizando"
                })
                
                total_chunks += info.points_count
                
            except Exception as e:
                logger.warning(f"Error getting info for collection {collection.name}: {str(e)}")
                collections_data.append({
                    "Nombre de Colección": collection.name,
                    "Cantidad de Puntos": "Error",
                    "Cantidad de Vectores": "Error", 
                    "Estado": "Error",
                    "Estado del Optimizador": "Error"
                })
        
        collections_df = pd.DataFrame(collections_data)
        
        status_msg = f"✅ Se encontraron {len(collections.collections)} colecciones con {total_chunks} chunks totales"
        
        return status_msg, collections_df
        
    except Exception as e:
        logger.error(f"Error getting collections summary: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

def delete_collection(collection_name: str) -> str:
    """
    Delete a specific collection using Qdrant native API.
    
    Args:
        collection_name: Name of the collection to delete
        
    Returns:
        Status message
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado"
        
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(url=settings.QDRANT_URL, api_key=settings.QDRANT_API_KEY)
        else:
            client = QdrantClient(url=settings.QDRANT_URL)
        
        # Delete the collection
        result = client.delete_collection(collection_name)
        
        logger.info(f"Collection '{collection_name}' deleted successfully")
        return f"✅ Colección '{collection_name}' eliminada exitosamente"
        
    except Exception as e:
        logger.error(f"Error deleting collection {collection_name}: {str(e)}")
        return f"❌ Error eliminando colección '{collection_name}': {str(e)}"

def delete_all_collections() -> str:
    """
    Delete all collections using Qdrant native API.
    
    Returns:
        Status message
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado"
        
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(url=settings.QDRANT_URL, api_key=settings.QDRANT_API_KEY)
        else:
            client = QdrantClient(url=settings.QDRANT_URL)
        
        # Get all collections first
        collections = client.get_collections()
        
        deleted_count = 0
        errors = []
        
        for collection in collections.collections:
            try:
                client.delete_collection(collection.name)
                deleted_count += 1
                logger.info(f"Deleted collection: {collection.name}")
            except Exception as e:
                error_msg = f"Falló eliminar {collection.name}: {str(e)}"
                errors.append(error_msg)
                logger.error(error_msg)
        
        if errors:
            return f"⚠️ Se eliminaron {deleted_count} colecciones, pero {len(errors)} fallaron: {'; '.join(errors)}"
        else:
            return f"✅ Se eliminaron exitosamente todas las {deleted_count} colecciones"
        
    except Exception as e:
        logger.error(f"Error deleting all collections: {str(e)}")
        return f"❌ Error eliminando todas las colecciones: {str(e)}"

def get_system_status() -> Tuple[str, pd.DataFrame]:
    """
    Get current system status including collections info.
    
    Returns:
        Tuple of (status_message, status_dataframe)
    """
    try:
        # Get collection statistics
        status_data = []
        
        for method in settings.CHUNKING_METHODS:
            collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method}"
            try:
                status_data.append({
                    "Colección": collection_name,
                    "Método": method.capitalize(),
                    "Estado": "Disponible",
                    "Descripción": f"Colección para el método de chunking {method}"
                })
            except Exception:
                status_data.append({
                    "Colección": collection_name,
                    "Método": method.capitalize(), 
                    "Estado": "No Disponible",
                    "Descripción": f"Colección para el método de chunking {method} (no creada)"
                })
        
        status_df = pd.DataFrame(status_data)
        
        # System configuration info
        config_info = f"""
        📊 **Configuración del Sistema:**
        - Modelo de Embeddings: {settings.OPENAI_EMBEDDING_MODEL}
        - Dimensiones de Embedding: {settings.EMBEDDING_DIMENSIONS}
        - URL de Qdrant: {settings.QDRANT_URL}
        - Top K Resultados: {settings.TOP_K_RESULTS}
        - Puntuación Mín. de Similitud: {settings.MIN_SIMILARITY_SCORE}
        - Métodos de Chunking Disponibles: {', '.join(settings.CHUNKING_METHODS)}
        """
        
        return config_info, status_df
        
    except Exception as e:
        logger.error(f"Error getting system status: {str(e)}")
        return f"❌ Error obteniendo estado del sistema: {str(e)}", pd.DataFrame()

# ================================
# Explainability helpers
# ================================

def _tokenize(text: str) -> list:
    if not text:
        return []
    return re.findall(r"\w+", text.lower())

def _coverage_ratio(query_tokens: list, chunk_tokens: list) -> float:
    if not query_tokens:
        return 0.0
    qset = set(query_tokens)
    cset = set(chunk_tokens)
    return len(qset & cset) / max(1, len(qset))

def _idf_mean_for_query(texts: list, query_tokens: list) -> Optional[float]:
    try:
        if not texts or not query_tokens:
            return None
        vectorizer = TfidfVectorizer(lowercase=True, token_pattern=r"\b\w+\b")
        vectorizer.fit(texts)
        idf_map = {term: idf for term, idf in zip(vectorizer.get_feature_names_out(), vectorizer.idf_)}
        vals = [idf_map[t] for t in set(query_tokens) if t in idf_map]
        if not vals:
            return None
        return float(np.mean(vals))
    except Exception:
        return None

def _truncate_snippet(s: str, max_len: int = 500) -> str:
    s = (s or "").strip().replace("\n", " ")
    return s[:max_len] + ("..." if len(s) > max_len else "")

def _enforce_combined_char_limit(examples_by_method: Dict[str, list], limit: int = 3000) -> Dict[str, list]:
    # Flatten, trim from the end until under limit
    def total_chars():
        return sum(len(e.get("snippet", "")) for lst in examples_by_method.values() for e in lst)
    if total_chars() <= limit:
        return examples_by_method
    # Greedy trim: reduce longest snippets first
    while total_chars() > limit:
        # find example with max length
        max_m, max_i, max_len = None, None, 0
        for m, lst in examples_by_method.items():
            for i, e in enumerate(lst):
                l = len(e.get("snippet", ""))
                if l > max_len:
                    max_len = l; max_m = m; max_i = i
        if max_m is None:
            break
        snip = examples_by_method[max_m][max_i]["snippet"]
        examples_by_method[max_m][max_i]["snippet"] = _truncate_snippet(snip, max(50, len(snip) - 100))
    return examples_by_method

def prepare_explainability_payload(query: str, results_value: Any) -> Tuple[dict, pd.DataFrame]:
    # Normalize results to DataFrame with expected headers
    expected_cols = ["Rank", "Similarity Score", "Chunking Method", "Text Preview", "Full Text"]
    df: pd.DataFrame
    if isinstance(results_value, pd.DataFrame):
        df = results_value.copy()
    elif isinstance(results_value, list):
        # try list of dicts first
        if results_value and isinstance(results_value[0], dict):
            df = pd.DataFrame(results_value)
        else:
            df = pd.DataFrame(results_value, columns=expected_cols[:len(results_value[0]) if results_value else 0])
    else:
        df = pd.DataFrame(columns=expected_cols)

    if df.empty:
        return {}, df

    # Ensure required columns exist
    for col in expected_cols:
        if col not in df.columns:
            df[col] = None

    # Group by method
    grouped = df.groupby("Chunking Method")

    # Build global texts for IDF
    all_texts = [str(t) for t in df["Full Text"].fillna("").tolist()]
    query_tokens = _tokenize(query)

    payload = {"query_text": query, "methods": []}

    examples_by_method: Dict[str, list] = {}

    for method, g in grouped:
        # metrics
        cos_vals = pd.to_numeric(g["Similarity Score"], errors="coerce").fillna(0.0).tolist()
        avg_cosine = float(np.mean(cos_vals)) if cos_vals else 0.0
        avg_bm25 = None  # not available

        # coverage per chunk
        coverages = []
        lengths = []
        snippets = []

        # pick top 3 by Similarity Score
        g_sorted = g.sort_values(by="Similarity Score", ascending=False).head(3)
        for _, row in g_sorted.iterrows():
            text = str(row.get("Full Text", ""))
            tokens = _tokenize(text)
            cov = _coverage_ratio(query_tokens, tokens)
            coverages.append(cov)
            lengths.append(len(tokens))
            snippets.append({
                "snippet": _truncate_snippet(text, 500),
                "similarity": round(float(row.get("Similarity Score", 0.0)), 3),
                "coverage": round(float(cov), 3),
            })

        avg_cov = float(np.mean(coverages)) if coverages else 0.0
        avg_len = float(np.mean(lengths)) if lengths else 0.0
        idf_mean = _idf_mean_for_query(all_texts, query_tokens)

        examples_by_method[method] = snippets

        payload["methods"].append({
            "method": str(method).lower(),
            "avg_cosine": round(avg_cosine, 3),
            "avg_bm25": (round(avg_bm25, 3) if isinstance(avg_bm25, (int, float)) else None),
            "coverage_ratio": round(avg_cov, 3),
            "idf_mean": (round(idf_mean, 3) if isinstance(idf_mean, (int, float)) else "informacion insuficiente"),
            "avg_chunk_length": round(avg_len, 1),
            "examples": snippets,
        })

    # Enforce combined snippet char limit
    examples_by_method = _enforce_combined_char_limit(examples_by_method, 3000)
    # Replace examples in payload with possibly trimmed versions
    for m in payload["methods"]:
        m_examples = examples_by_method.get(m["method"], m.get("examples", []))
        m["examples"] = m_examples

    return payload, df

def explain_ranking(query: str, results_value: Any) -> Tuple[str, str, pd.DataFrame, str]:
    try:
        if not query or not str(query).strip():
            return ("❌ No query provided.", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), "")

        payload, df = prepare_explainability_payload(query, results_value)
        if not payload or df.empty:
            return ("⚠️ No hay resultados para explicar. Realiza una búsqueda primero.", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), "")

        # Build prompt messages
        system_msg = (
            "Eres un analista experto en Recuperacion de Informacion y en Explicabilidad de sistemas de embeddings. "
            "Has sido formado en el MIT y tu estilo es tecnico, preciso y pedagogico. "
            "Tu tarea es analizar, de forma global y comparativa, los resultados de un sistema de busqueda que usa distintos "
            "metodos de chunking (fixed, sentence, adaptive). "
            "IMPORTANTE: No solo compares metricas. Explica pedagogicamente COMO FUNCIONA cada metodo, sus ventajas teoricas, "
            "limitaciones intrinsecas y por que el patron observado es consistente con la teoria. "
            "Reglas: No inventes informacion; no modifiques valores; si faltan datos, dilo; tono tecnico; concision; "
            "apoya con datos y snippets; indica cuando diferencias sean pequenas. "
            "FORMATO: Devuelve (a) resumen breve (max 4 frases) explicando tendencias y por que, "
            "(b) un JSON con estructura exacta: "
            "{\"summary\": \"\", \"per_method\": [{\"method\": \"\", \"reasoning\": \"\", \"evidence\": \"\", \"risks\": \"\"}], \"recommendation\": \"\"}"
        )

        user_msg = (
            "Analiza los siguientes resultados globales obtenidos tras una busqueda. Cada metodo de chunking incluye medidas numericas agregadas y fragmentos representativos. "
            "INSTRUCCIONES ESPECIFICAS: "
            "1) EXPLICA COMO FUNCIONA cada metodo teoricamente (fixed: tamano fijo + overlap; sentence: agrupa oraciones naturales; adaptive: similitud semantica entre oraciones). "
            "2) CONECTA la teoria con los resultados observados (ej: 'fixed tiende a chunks uniformes pero puede cortar conceptos'). "
            "3) USA las metricas para validar/refutar expectativas teoricas. "
            "4) CITA frases especificas de los snippets como evidencia. "
            "5) Si las diferencias son pequenas, explicalo y sugiere factores que podrian ser mas determinantes. "
            "Usa estilo tecnico y pedagogico (nivel MIT-PhD pero entendible). No generes texto fuera del material proporcionado.\n\n"
            f"QUERY:\n{payload['query_text']}\n\nRESULTADOS (JSON):\n{json.dumps(payload, ensure_ascii=False)}\n\n"
            "Cada metodo contiene: avg_cosine (similitud coseno media), coverage_ratio (% terminos query en chunks), "
            "idf_mean (peso IDF medio de terminos), avg_chunk_length (longitud promedio), examples (snippets con metricas).\n\n"
            "TASK DETALLADA: "
            "1) Tendencias globales y EXPLICACION TEORICA de cada metodo. "
            "2) Factores numericos que explican diferencias + conexion con principios de cada metodo. "
            "3) Frases de snippets que ilustran fortalezas/debilidades de cada enfoque. "
            "4) Riesgos inherentes a cada metodo (dilucion, perdida contexto, fronteras artificiales, etc.). "
            "5) Recomendacion fundamentada para este tipo de query y corpus.\n"
            "FORMATO: resumen tecnico (max 4 frases) + JSON valido con estructura exacta. Si falta info, usa 'informacion insuficiente'."
        )

        client = _get_openai_client()
        completion = client.chat.completions.create(
            model="gpt-4o-mini",
            temperature=0.3,
            max_tokens=1000,
            messages=[
                {"role": "system", "content": system_msg},
                {"role": "user", "content": user_msg},
            ],
        )
        content = completion.choices[0].message.content if completion and completion.choices else ""

        # Try to parse JSON from content
        summary_text = ""
        json_obj = None
        # Heuristic: find JSON block
        if content:
            try:
                # Direct parse
                json_obj = json.loads(content)
            except Exception:
                # Extract first {...} block
                start = content.find("{")
                end = content.rfind("}")
                if start != -1 and end != -1 and end > start:
                    try:
                        json_obj = json.loads(content[start:end+1])
                    except Exception:
                        json_obj = None

        if not json_obj:
            return (
                "⚠️ No se pudo generar la explicación en formato JSON válido.",
                content[:800] if content else "",
                pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]),
                "",
            )

        summary_text = str(json_obj.get("summary", ""))
        per_method = json_obj.get("per_method", []) or []
        recommendation = str(json_obj.get("recommendation", ""))

        table_rows = []
        for item in per_method:
            table_rows.append({
                "Method": item.get("method", ""),
                "Reasoning": item.get("reasoning", ""),
                "Evidence": item.get("evidence", ""),
                "Risks": item.get("risks", ""),
            })
        table_df = pd.DataFrame(table_rows, columns=["Method","Reasoning","Evidence","Risks"])

        return ("", summary_text, table_df, recommendation)

    except Exception as e:
        logger.error(f"Explainability error: {str(e)}")
        return (
            "❌ No se pudo generar la explicación. Verifica tu clave de API o la conexión.",
            "",
            pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]),
            "",
        )
    """
    Get current system status including collections info.
    
    Returns:
        Tuple of (status_message, status_dataframe)
    """
    try:
        # Get collection statistics
        status_data = []
        total_chunks = 0
        
        for method in settings.CHUNKING_METHODS:
            collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method}"
            try:
                # This would need a method to get collection info
                # For now, we'll show configured methods
                status_data.append({
                    "Collection": collection_name,
                    "Method": method,
                    "Status": "Available",
                    "Description": f"Collection for {method} chunking method"
                })
            except Exception:
                status_data.append({
                    "Collection": collection_name,
                    "Method": method,
                    "Status": "Not Available",
                    "Description": f"Collection for {method} chunking method (not created)"
                })
        
        status_df = pd.DataFrame(status_data)
        
        # System configuration info
        config_info = f"""
        📊 **System Configuration:**
        - Embedding Model: {settings.OPENAI_EMBEDDING_MODEL}
        - Embedding Dimensions: {settings.EMBEDDING_DIMENSIONS}
        - Qdrant URL: {settings.QDRANT_URL}
        - Top K Results: {settings.TOP_K_RESULTS}
        - Min Similarity Score: {settings.MIN_SIMILARITY_SCORE}
        - Available Chunking Methods: {', '.join(settings.CHUNKING_METHODS)}
        """
        
        return config_info, status_df
        
    except Exception as e:
        logger.error(f"Error getting system status: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

# Create Gradio interface
with gr.Blocks(title="Sistema de Chunking y Búsqueda de Documentos", theme=gr.themes.Soft(), css="""
    .gradio-container {
        max-width: 1200px !important;
    }
    .dataframe table {
        table-layout: fixed !important;
        word-wrap: break-word !important;
    }
    .dataframe td {
        white-space: pre-wrap !important;
        vertical-align: top !important;
        padding: 8px !important;
    }
""") as demo:
    gr.Markdown("""
    # 📄 Sistema de Chunking y Búsqueda de Documentos
    
    Este sistema procesa documentos utilizando múltiples estrategias de chunking y permite búsqueda semántica usando embeddings de OpenAI almacenados en Qdrant.
    
    ## Métodos de Chunking Disponibles:
    - **Fixed**: Chunks de tamaño fijo con solapamiento
    - **Sentence**: Chunking basado en oraciones
    - **Adaptive**: Chunking adaptativo basado en la estructura del contenido
    """)
    
    with gr.Tab("📤 Procesamiento de Documentos"):
        gr.Markdown("### Cargar y procesar documentos")
        
        with gr.Row():
            with gr.Column():
                file_input = gr.File(
                    label="Cargar Documento", 
                    file_types=[".pdf", ".txt", ".docx"],
                    type="filepath"
                )
                
                chunking_methods = gr.CheckboxGroup(
                    choices=settings.CHUNKING_METHODS,
                    value=settings.CHUNKING_METHODS,  # Select all by default
                    label="Seleccionar Métodos de Chunking"
                )
                
                process_btn = gr.Button("🔄 Procesar Documento", variant="primary")
            
            with gr.Column():
                process_status = gr.Textbox(
                    label="Estado del Procesamiento",
                    placeholder="Carga un documento y haz clic en 'Procesar Documento'",
                    interactive=False
                )
        
        chunks_table = gr.Dataframe(
            label="Resumen de Chunks",
            headers=["Método de Chunking", "Número de Chunks", "Longitud Promedio de Chunk", "Chunk de Muestra"],
            interactive=False,
            wrap=True,
            column_widths=["15%", "15%", "15%", "55%"],
            max_height=400
        )
        
        process_btn.click(
            process_document,
            inputs=[file_input, chunking_methods],
            outputs=[process_status, chunks_table]
        )
    
    with gr.Tab("🔍 Búsqueda"):
        gr.Markdown("### Buscar contenido similar en todos los documentos procesados")
        
        with gr.Row():
            with gr.Column():
                search_input = gr.Textbox(
                    label="Consulta de Búsqueda",
                    placeholder="Ingresa tu consulta de búsqueda aquí...",
                    lines=2
                )
                
                top_k_input = gr.Slider(
                    minimum=1,
                    maximum=20,
                    value=settings.TOP_K_RESULTS,
                    step=1,
                    label="Número de Resultados"
                )
                
                search_btn = gr.Button("🔍 Buscar", variant="primary")
            
            with gr.Column():
                search_status = gr.Textbox(
                    label="Estado de la Búsqueda",
                    placeholder="Ingresa una consulta y haz clic en 'Buscar'",
                    interactive=False
                )
        
        results_table = gr.Dataframe(
            label="Resultados de Búsqueda",
            headers=["Rango", "Puntuación de Similitud", "Método de Chunking", "Vista Previa de Texto", "Texto Completo"],
            interactive=False,
            wrap=True,
            column_widths=["8%", "12%", "15%", "30%", "35%"],
            max_height=600
        )
        
        # Explainability panel
        gr.Markdown("### 🧠 Explicabilidad de los resultados")
        with gr.Row():
            with gr.Column(scale=1):
                explain_btn = gr.Button("🧠 Explicabilidad de los resultados", variant="secondary", interactive=False)
                explain_status = gr.Textbox(label="Estado", interactive=False)
            with gr.Column(scale=2):
                explain_summary = gr.Textbox(label="Resumen técnico", lines=4, interactive=False)
                per_method_table = gr.Dataframe(
                    label="Análisis por método",
                    headers=["Method", "Reasoning", "Evidence", "Risks"],
                    interactive=False,
                    wrap=True,
                    max_height=300
                )
                recommendation_text = gr.Textbox(label="Recomendación", lines=2, interactive=False)

        # Search action
        search_event = search_btn.click(
            search_chunks,
            inputs=[search_input, top_k_input],
            outputs=[search_status, results_table]
        )

        # After search: enable explain button and clear previous explanation
        search_event.then(
            lambda: gr.update(interactive=True),
            inputs=None,
            outputs=explain_btn
        ).then(
            lambda: ("", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), ""),
            inputs=None,
            outputs=[explain_status, explain_summary, per_method_table, recommendation_text]
        )

        # Explain action
        explain_btn.click(
            explain_ranking,
            inputs=[search_input, results_table],
            outputs=[explain_status, explain_summary, per_method_table, recommendation_text]
        )
    
    with gr.Tab("⚙️ Estado del Sistema"):
        gr.Markdown("### Configuración y estado actual del sistema")
        
        status_btn = gr.Button("🔄 Actualizar Estado", variant="secondary")
        
        system_info = gr.Textbox(
            label="Información del Sistema",
            lines=10,
            interactive=False
        )
        
        collections_table = gr.Dataframe(
            label="Collections Status",
            headers=["Colección", "Método", "Estado", "Descripción"],
            interactive=False,
            wrap=True,
            column_widths=["25%", "15%", "15%", "45%"],
            max_height=300
        )
        
        # Load initial status
        demo.load(
            get_system_status,
            outputs=[system_info, collections_table]
        )
        
        status_btn.click(
            get_system_status,
            outputs=[system_info, collections_table]
        )
    
    # Data Management Tab
    with gr.Tab("🗄️ Gestión de Datos"):
        gr.Markdown("## Gestión de Base de Datos Qdrant")
        gr.Markdown("Gestiona tus colecciones de documentos y visualiza datos usando las herramientas nativas de Qdrant.")
        
        with gr.Row():
            with gr.Column():
                gr.Markdown("### 📊 Visualización")
                dashboard_btn = gr.Button("🌐 Obtener Acceso al Dashboard", variant="primary", size="lg")
                dashboard_output = gr.Textbox(
                    label="Información de Acceso al Dashboard de Qdrant",
                    placeholder="Haz clic en el botón para obtener la URL del dashboard y detalles de acceso",
                    lines=15,
                    interactive=False
                )
                
            with gr.Column():
                gr.Markdown("### 📋 Resumen de Colecciones")
                summary_btn = gr.Button("📊 Obtener Resumen de Colecciones", variant="secondary")
                summary_output = gr.Textbox(
                    label="Información de Colecciones",
                    lines=8,
                    interactive=False
                )
                summary_table = gr.Dataframe(
                    label="Detalles de Colecciones",
                    headers=["Nombre de Colección", "Cantidad de Puntos", "Cantidad de Vectores", "Estado", "Estado del Optimizador"],
                    interactive=False,
                    wrap=True,
                    max_height=200
                )
        
        gr.Markdown("### 🗑️ Gestión de Colecciones")
        with gr.Row():
            with gr.Column():
                gr.Markdown("**Eliminar Colección Específica:**")
                method_dropdown = gr.Dropdown(
                    choices=settings.CHUNKING_METHODS,
                    label="Seleccionar Método de Chunking",
                    value=settings.CHUNKING_METHODS[0] if settings.CHUNKING_METHODS else None
                )
                delete_specific_btn = gr.Button("🗑️ Eliminar Colección Seleccionada", variant="secondary")
                
            with gr.Column():
                gr.Markdown("**Eliminar Todas las Colecciones:**")
                gr.Markdown("⚠️ **Advertencia**: ¡Esto eliminará TODAS las colecciones permanentemente!")
                delete_all_btn = gr.Button("🗑️ Eliminar Todas las Colecciones", variant="stop")

        with gr.Row():
            with gr.Column():
                gr.Markdown("**Reiniciar Seguimiento de Documentos:**")
                gr.Markdown("🔄 Forzar limpieza de colecciones en la próxima carga de documento (útil cuando se reprocesa el mismo archivo)")
                reset_tracking_btn = gr.Button("🔄 Reiniciar Seguimiento de Documentos", variant="secondary")
        
        delete_output = gr.Textbox(
            label="Estado de Eliminación",
            lines=3,
            interactive=False
        )
        
        # Event handlers for Data Management
        dashboard_btn.click(
            get_qdrant_dashboard_url,
            outputs=dashboard_output
        )
        
        summary_btn.click(
            get_collections_summary,
            outputs=[summary_output, summary_table]
        )
        
        def _delete_by_method(method_name: str) -> str:
            try:
                if not method_name:
                    return "❌ Por favor selecciona un método"
                collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method_name}"
                return delete_collection(collection_name)
            except Exception as e:
                return f"❌ Error eliminando: {str(e)}"

        delete_specific_btn.click(
            _delete_by_method,
            inputs=method_dropdown,
            outputs=delete_output
        )
        
        delete_all_btn.click(
            delete_all_collections,
            outputs=delete_output
        )
        
        reset_tracking_btn.click(
            reset_document_tracking,
            outputs=delete_output
        )

if __name__ == "__main__":
    # Set up environment for deployment
    port = int(os.environ.get("PORT", 7860))
    
    print("🚀 Iniciando Sistema de Chunking y Búsqueda de Documentos...")
    
    # Initialize components with error handling
    init_success = initialize_components()
    
    if not init_success:
        print("⚠️  Advertencia: Algunos componentes fallaron al inicializar. Verifica la configuración.")
        print("📋 Asegúrate de tener:")
        print("   - Variable de entorno OPENAI_API_KEY configurada")
        print("   - Instancia de Qdrant ejecutándose (QDRANT_URL configurado)")
    else:
        print("✅ Todos los componentes inicializados exitosamente")
    
    print(f"📊 Configurado para {len(settings.CHUNKING_METHODS)} métodos de chunking")
    print(f"🔗 URL de Qdrant: {settings.QDRANT_URL}")
    print(f"🤖 Modelo de Embeddings: {settings.OPENAI_EMBEDDING_MODEL}")
    
    demo.launch(
        server_name="0.0.0.0",
        server_port=port,
        share=False  # Gradio Spaces will handle public access
    )