Blame for app.py

import gradio as gr
import logging
import os
import hashlib
from typing import List, Tuple, Optional, Dict, Any
import pandas as pd
from qdrant_client import QdrantClient
import re
import json
from collections import defaultdict, Counter
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from openai import OpenAI

# Import your existing modules
from document_processor import DocumentProcessor
from embeddings_qdrant import EmbeddingsQdrantPipeline
import settings

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Global variables for components (will be initialized in main)
document_processor = None
embeddings_pipeline = None

# Global variable to track current document identity
current_document_hash = None

# OpenAI client (lazy init)
_openai_client: Optional[OpenAI] = None

def _get_openai_client() -> OpenAI:
    global _openai_client
    if _openai_client is None:
        api_key = getattr(settings, "OPENAI_API_KEY", os.getenv("OPENAI_API_KEY"))
        _openai_client = OpenAI(api_key=api_key)
    return _openai_client

def get_file_hash(file_path: str) -> str:
    """
    Calculate MD5 hash of a file to track document identity.
    
    Args:
        file_path: Path to the file
        
    Returns:
        MD5 hash string of the file content
    """
    hash_md5 = hashlib.md5()
    try:
        with open(file_path, "rb") as f:
            for chunk in iter(lambda: f.read(4096), b""):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()
    except Exception as e:
        logger.error(f"Error calculating file hash for {file_path}: {str(e)}")
        return str(hash(file_path))  # Fallback to simple hash of path

def clear_all_collections_for_new_document() -> bool:
    """
    Clear all collections in Qdrant when a new document is detected.
    
    Returns:
        bool: True if clearing was successful or not needed, False on error
    """
    try:
        logger.info("🗑️ Nuevo documento detectado: limpiando todas las colecciones existentes...")
        result = delete_all_collections()
        if "✅" in result:
            logger.info("✅ Todas las colecciones fueron eliminadas exitosamente")
            return True
        else:
            logger.warning(f"⚠️ Limpieza de colecciones completada con advertencias: {result}")
            return True
    except Exception as e:
        logger.error(f"❌ Error al limpiar colecciones para nuevo documento: {str(e)}")
        return False

def reset_document_tracking() -> str:
    """
    Reset the document tracking hash to force collection clearing on next upload.
    
    Returns:
        Status message
    """
    global current_document_hash
    current_document_hash = None
    logger.info("🔄 Document tracking reset - next upload will clear all collections")
    return "✅ Seguimiento de documentos reiniciado. La próxima carga de documento eliminará todas las colecciones existentes."

def initialize_components():
    """Initialize the pipeline components with error handling for cloud deployment."""
    global document_processor, embeddings_pipeline
    
    try:
        document_processor = DocumentProcessor()
        logger.info("✅ Document processor initialized")
    except Exception as e:
        logger.error(f"❌ Failed to initialize document processor: {str(e)}")
        document_processor = None
    
    try:
        embeddings_pipeline = EmbeddingsQdrantPipeline()
        logger.info("✅ Embeddings pipeline initialized")
    except Exception as e:
        logger.error(f"❌ Failed to initialize embeddings pipeline: {str(e)}")
        embeddings_pipeline = None
        
    return document_processor is not None and embeddings_pipeline is not None

def process_document(file_path: str, chunking_methods: List[str]) -> Tuple[str, pd.DataFrame]:
    """
    Process a document with selected chunking methods.
    Automatically clears all collections if a different document is detected.
    
    Args:
        file_path: Path to the uploaded file
        chunking_methods: List of selected chunking methods
    
    Returns:
        Tuple of (status_message, chunks_dataframe)
    """
    global current_document_hash
    
    try:
        if document_processor is None:
            return "❌ Error: Procesador de documentos no inicializado. Verifica la configuración del sistema.", pd.DataFrame()
            
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado. Verifica la conexión a Qdrant y la clave API de OpenAI.", pd.DataFrame()
            
        if not file_path:
            return "❌ Error: No se ha cargado ningún archivo", pd.DataFrame()
        
        if not chunking_methods:
            return "❌ Error: Por favor selecciona al menos un método de chunking", pd.DataFrame()
        
        # Calculate hash of the current document
        new_document_hash = get_file_hash(file_path)
        logger.info(f"Document hash: {new_document_hash}")
        
        # Check if this is a different document than the previous one
        if current_document_hash is not None and current_document_hash != new_document_hash:
            logger.info(f"🔄 Documento diferente detectado (hash anterior: {current_document_hash[:8]}..., nuevo: {new_document_hash[:8]}...)")
            
            # Clear all collections for the new document
            clear_success = clear_all_collections_for_new_document()
            if not clear_success:
                logger.warning("⚠️ Falló la limpieza de colecciones, pero continuando con el procesamiento...")
        elif current_document_hash is None:
            logger.info("📄 Primer documento cargado en esta sesión")
        else:
            logger.info("📄 Mismo documento que el anterior, manteniendo embeddings existentes")
        
        # Update the current document hash
        current_document_hash = new_document_hash
        
        # Process the document
        logger.info(f"Processing document: {file_path}")
        chunks_by_method = document_processor.process_document(
            file_path=file_path,
            chunking_methods=chunking_methods
        )
        
        if not chunks_by_method:
            return "❌ Error: No se generaron chunks del documento", pd.DataFrame()
        
        # Store embeddings in Qdrant
        success = embeddings_pipeline.process_and_store_chunks(chunks_by_method)
        
        if not success:
            return "❌ Error: Falló el almacenamiento de embeddings en Qdrant", pd.DataFrame()
        
        # Create a summary dataframe
        summary_data = []
        total_chunks = 0
        for method, chunks in chunks_by_method.items():
            # Use a shorter sample chunk for better display
            sample_text = chunks[0][:150] + "..." if chunks and len(chunks[0]) > 150 else (chunks[0] if chunks else "Sin chunks")
            summary_data.append({
                "Método de Chunking": method.capitalize(),
                "Número de Chunks": len(chunks),
                "Longitud Promedio de Chunk": round(sum(len(chunk) for chunk in chunks) / len(chunks)) if chunks else 0,
                "Chunk de Muestra": sample_text
            })
            total_chunks += len(chunks)
        
        summary_df = pd.DataFrame(summary_data)
        
        status_msg = f"✅ ¡Éxito! Documento procesado con {len(chunking_methods)} métodos. Total de chunks: {total_chunks}"
        
        return status_msg, summary_df
        
    except Exception as e:
        logger.error(f"Error processing document: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()
        
    except Exception as e:
        logger.error(f"Error processing document: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

def search_chunks(query: str, top_k: int = None) -> Tuple[str, pd.DataFrame]:
    """
    Search for similar chunks across all collections.
    
    Args:
        query: Search query
        top_k: Number of results to return (uses settings default if None)
    
    Returns:
        Tuple of (status_message, results_dataframe)
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado. Verifica la conexión a Qdrant y la clave API de OpenAI.", pd.DataFrame()
            
        if not query.strip():
            return "❌ Error: Por favor ingresa una consulta de búsqueda", pd.DataFrame()
        
        # Use provided top_k or default from settings
        k = top_k if top_k is not None else settings.TOP_K_RESULTS
        
        # Perform search
        logger.info(f"Searching for: '{query}' (top_k={k})")
        results = embeddings_pipeline.search_chunks(query=query, top_k=k)
        
        if not results:
            return "⚠️ No se encontraron resultados. Prueba una consulta diferente o verifica si los documentos están procesados.", pd.DataFrame()
        
        # Create results dataframe
        results_data = []
        for i, (text, score, method) in enumerate(results, 1):
            # Better text preview with line breaks preserved
            text_preview = text[:300] + "..." if len(text) > 300 else text
            # Clean up text preview for better display
            text_preview = text_preview.replace('\n', ' ').replace('\r', ' ').strip()
            
            results_data.append({
                "Rango": i,
                "Puntuación de Similitud": round(score, 4),
                "Método de Chunking": method.capitalize(),
                "Vista Previa de Texto": text_preview,
                "Texto Completo": text
            })
        
        results_df = pd.DataFrame(results_data)
        
        status_msg = f"✅ Se encontraron {len(results)} resultados para la consulta: '{query}'"
        
        return status_msg, results_df
        
    except Exception as e:
        logger.error(f"Error searching chunks: {str(e)}")
        return f"❌ Error en la búsqueda: {str(e)}", pd.DataFrame()

def get_qdrant_dashboard_url() -> str:
    """
    Get the Qdrant dashboard URL with authentication for public access.
    
    Returns:
        Dashboard URL with access instructions
    """
    try:
        # For Qdrant Cloud, the dashboard is usually at the same URL
        base_url = settings.QDRANT_URL.replace(':6333', '').replace(':6334', '')
        dashboard_url = f"{base_url}/dashboard"
        
        return f"""🌐 **Acceso al Dashboard de Qdrant:**

**🔗 URL Directo:** {dashboard_url}

🔑 **Autenticación (si es necesaria):**
- Clave API: `{settings.QDRANT_API_KEY}`
- URL del Servidor: `{settings.QDRANT_URL}`

📋 **Cómo Acceder:**
1. Haz clic/copia la URL de arriba para abrir el dashboard de Qdrant
2. Si se solicita autenticación, usa la clave API proporcionada arriba
3. Explora tus colecciones, vectores y datos visualmente

⚡ **Lo que puedes hacer:**
- 📊 Ver estadísticas y metadatos de colecciones
- 🔍 Explorar chunks de documentos almacenados
- 📈 Visualizar vectores de embeddings y relaciones
- ⚙️ Monitorear rendimiento y salud del cluster
- 🔎 Buscar y filtrar tus datos interactivamente

💡 **Tip:** Bookmark this URL for quick access to your vector database!"""
        
    except Exception as e:
        logger.error(f"Error generating dashboard URL: {str(e)}")
        return f"❌ Error generating dashboard URL: {str(e)}"

def get_collections_summary() -> Tuple[str, pd.DataFrame]:
    """
    Get summary of all Qdrant collections using native APIs.
    
    Returns:
        Tuple of (status_message, collections_dataframe)
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado", pd.DataFrame()
        
        # Use Qdrant native API to get collections info
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication and extended timeout for HF Spaces
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(
                url=settings.QDRANT_URL, 
                api_key=settings.QDRANT_API_KEY,
                timeout=60.0,
                prefer_grpc=False
            )
        else:
            client = QdrantClient(
                url=settings.QDRANT_URL,
                timeout=60.0,
                prefer_grpc=False
            )
        
        # Get all collections
        collections = client.get_collections()
        
        collections_data = []
        total_chunks = 0
        
        for collection in collections.collections:
            try:
                # Get detailed info for each collection
                info = client.get_collection(collection.name)
                
                collections_data.append({
                    "Nombre de Colección": collection.name,
                    "Cantidad de Puntos": info.points_count,
                    "Cantidad de Vectores": info.vectors_count or info.points_count,
                    "Estado": info.status.value if hasattr(info.status, 'value') else str(info.status),
                    "Estado del Optimizador": "OK" if info.optimizer_status else "Optimizando"
                })
                
                total_chunks += info.points_count
                
            except Exception as e:
                logger.warning(f"Error getting info for collection {collection.name}: {str(e)}")
                collections_data.append({
                    "Nombre de Colección": collection.name,
                    "Cantidad de Puntos": "Error",
                    "Cantidad de Vectores": "Error", 
                    "Estado": "Error",
                    "Estado del Optimizador": "Error"
                })
        
        collections_df = pd.DataFrame(collections_data)
        
        status_msg = f"✅ Se encontraron {len(collections.collections)} colecciones con {total_chunks} chunks totales"
        
        return status_msg, collections_df
        
    except Exception as e:
        logger.error(f"Error getting collections summary: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

def delete_collection(collection_name: str) -> str:
    """
    Delete a specific collection using Qdrant native API.
    
    Args:
        collection_name: Name of the collection to delete
        
    Returns:
        Status message
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado"
        
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(url=settings.QDRANT_URL, api_key=settings.QDRANT_API_KEY)
        else:
            client = QdrantClient(url=settings.QDRANT_URL)
        
        # Delete the collection
        result = client.delete_collection(collection_name)
        
        logger.info(f"Collection '{collection_name}' deleted successfully")
        return f"✅ Colección '{collection_name}' eliminada exitosamente"
        
    except Exception as e:
        logger.error(f"Error deleting collection {collection_name}: {str(e)}")
        return f"❌ Error eliminando colección '{collection_name}': {str(e)}"

def delete_all_collections() -> str:
    """
    Delete all collections using Qdrant native API.
    
    Returns:
        Status message
    """
    try:
        if embeddings_pipeline is None:
            return "❌ Error: Pipeline de embeddings no inicializado"
        
        from qdrant_client import QdrantClient
        
        # Create client with proper authentication
        if hasattr(settings, 'QDRANT_API_KEY') and settings.QDRANT_API_KEY:
            client = QdrantClient(url=settings.QDRANT_URL, api_key=settings.QDRANT_API_KEY)
        else:
            client = QdrantClient(url=settings.QDRANT_URL)
        
        # Get all collections first
        collections = client.get_collections()
        
        deleted_count = 0
        errors = []
        
        for collection in collections.collections:
            try:
                client.delete_collection(collection.name)
                deleted_count += 1
                logger.info(f"Deleted collection: {collection.name}")
            except Exception as e:
                error_msg = f"Falló eliminar {collection.name}: {str(e)}"
                errors.append(error_msg)
                logger.error(error_msg)
        
        if errors:
            return f"⚠️ Se eliminaron {deleted_count} colecciones, pero {len(errors)} fallaron: {'; '.join(errors)}"
        else:
            return f"✅ Se eliminaron exitosamente todas las {deleted_count} colecciones"
        
    except Exception as e:
        logger.error(f"Error deleting all collections: {str(e)}")
        return f"❌ Error eliminando todas las colecciones: {str(e)}"

def get_system_status() -> Tuple[str, pd.DataFrame]:
    """
    Get current system status including collections info.
    
    Returns:
        Tuple of (status_message, status_dataframe)
    """
    try:
        # Get collection statistics
        status_data = []
        
        for method in settings.CHUNKING_METHODS:
            collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method}"
            try:
                status_data.append({
                    "Colección": collection_name,
                    "Método": method.capitalize(),
                    "Estado": "Disponible",
                    "Descripción": f"Colección para el método de chunking {method}"
                })
            except Exception:
                status_data.append({
                    "Colección": collection_name,
                    "Método": method.capitalize(), 
                    "Estado": "No Disponible",
                    "Descripción": f"Colección para el método de chunking {method} (no creada)"
                })
        
        status_df = pd.DataFrame(status_data)
        
        # System configuration info
        config_info = f"""
        📊 **Configuración del Sistema:**
        - Modelo de Embeddings: {settings.OPENAI_EMBEDDING_MODEL}
        - Dimensiones de Embedding: {settings.EMBEDDING_DIMENSIONS}
        - URL de Qdrant: {settings.QDRANT_URL}
        - Top K Resultados: {settings.TOP_K_RESULTS}
        - Puntuación Mín. de Similitud: {settings.MIN_SIMILARITY_SCORE}
        - Métodos de Chunking Disponibles: {', '.join(settings.CHUNKING_METHODS)}
        """
        
        return config_info, status_df
        
    except Exception as e:
        logger.error(f"Error getting system status: {str(e)}")
        return f"❌ Error obteniendo estado del sistema: {str(e)}", pd.DataFrame()

# ================================
# Explainability helpers
# ================================

def _tokenize(text: str) -> list:
    if not text:
        return []
    return re.findall(r"\w+", text.lower())

def _coverage_ratio(query_tokens: list, chunk_tokens: list) -> float:
    if not query_tokens:
        return 0.0
    qset = set(query_tokens)
    cset = set(chunk_tokens)
    return len(qset & cset) / max(1, len(qset))

def _idf_mean_for_query(texts: list, query_tokens: list) -> Optional[float]:
    try:
        if not texts or not query_tokens:
            return None
        vectorizer = TfidfVectorizer(lowercase=True, token_pattern=r"\b\w+\b")
        vectorizer.fit(texts)
        idf_map = {term: idf for term, idf in zip(vectorizer.get_feature_names_out(), vectorizer.idf_)}
        vals = [idf_map[t] for t in set(query_tokens) if t in idf_map]
        if not vals:
            return None
        return float(np.mean(vals))
    except Exception:
        return None

def _truncate_snippet(s: str, max_len: int = 500) -> str:
    s = (s or "").strip().replace("\n", " ")
    return s[:max_len] + ("..." if len(s) > max_len else "")

def _enforce_combined_char_limit(examples_by_method: Dict[str, list], limit: int = 3000) -> Dict[str, list]:
    # Flatten, trim from the end until under limit
    def total_chars():
        return sum(len(e.get("snippet", "")) for lst in examples_by_method.values() for e in lst)
    if total_chars() <= limit:
        return examples_by_method
    # Greedy trim: reduce longest snippets first
    while total_chars() > limit:
        # find example with max length
        max_m, max_i, max_len = None, None, 0
        for m, lst in examples_by_method.items():
            for i, e in enumerate(lst):
                l = len(e.get("snippet", ""))
                if l > max_len:
                    max_len = l; max_m = m; max_i = i
        if max_m is None:
            break
        snip = examples_by_method[max_m][max_i]["snippet"]
        examples_by_method[max_m][max_i]["snippet"] = _truncate_snippet(snip, max(50, len(snip) - 100))
    return examples_by_method

def prepare_explainability_payload(query: str, results_value: Any) -> Tuple[dict, pd.DataFrame]:
    # Normalize results to DataFrame with expected headers
    expected_cols = ["Rank", "Similarity Score", "Chunking Method", "Text Preview", "Full Text"]
    df: pd.DataFrame
    if isinstance(results_value, pd.DataFrame):
        df = results_value.copy()
    elif isinstance(results_value, list):
        # try list of dicts first
        if results_value and isinstance(results_value[0], dict):
            df = pd.DataFrame(results_value)
        else:
            df = pd.DataFrame(results_value, columns=expected_cols[:len(results_value[0]) if results_value else 0])
    else:
        df = pd.DataFrame(columns=expected_cols)

    if df.empty:
        return {}, df

    # Ensure required columns exist
    for col in expected_cols:
        if col not in df.columns:
            df[col] = None

    # Group by method
    grouped = df.groupby("Chunking Method")

    # Build global texts for IDF
    all_texts = [str(t) for t in df["Full Text"].fillna("").tolist()]
    query_tokens = _tokenize(query)

    payload = {"query_text": query, "methods": []}

    examples_by_method: Dict[str, list] = {}

    for method, g in grouped:
        # metrics
        cos_vals = pd.to_numeric(g["Similarity Score"], errors="coerce").fillna(0.0).tolist()
        avg_cosine = float(np.mean(cos_vals)) if cos_vals else 0.0
        avg_bm25 = None  # not available

        # coverage per chunk
        coverages = []
        lengths = []
        snippets = []

        # pick top 3 by Similarity Score
        g_sorted = g.sort_values(by="Similarity Score", ascending=False).head(3)
        for _, row in g_sorted.iterrows():
            text = str(row.get("Full Text", ""))
            tokens = _tokenize(text)
            cov = _coverage_ratio(query_tokens, tokens)
            coverages.append(cov)
            lengths.append(len(tokens))
            snippets.append({
                "snippet": _truncate_snippet(text, 500),
                "similarity": round(float(row.get("Similarity Score", 0.0)), 3),
                "coverage": round(float(cov), 3),
            })

        avg_cov = float(np.mean(coverages)) if coverages else 0.0
        avg_len = float(np.mean(lengths)) if lengths else 0.0
        idf_mean = _idf_mean_for_query(all_texts, query_tokens)

        examples_by_method[method] = snippets

        payload["methods"].append({
            "method": str(method).lower(),
            "avg_cosine": round(avg_cosine, 3),
            "avg_bm25": (round(avg_bm25, 3) if isinstance(avg_bm25, (int, float)) else None),
            "coverage_ratio": round(avg_cov, 3),
            "idf_mean": (round(idf_mean, 3) if isinstance(idf_mean, (int, float)) else "informacion insuficiente"),
            "avg_chunk_length": round(avg_len, 1),
            "examples": snippets,
        })

    # Enforce combined snippet char limit
    examples_by_method = _enforce_combined_char_limit(examples_by_method, 3000)
    # Replace examples in payload with possibly trimmed versions
    for m in payload["methods"]:
        m_examples = examples_by_method.get(m["method"], m.get("examples", []))
        m["examples"] = m_examples

    return payload, df

def explain_ranking(query: str, results_value: Any) -> Tuple[str, str, pd.DataFrame, str]:
    try:
        if not query or not str(query).strip():
            return ("❌ No query provided.", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), "")

        payload, df = prepare_explainability_payload(query, results_value)
        if not payload or df.empty:
            return ("⚠️ No hay resultados para explicar. Realiza una búsqueda primero.", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), "")

        # Build prompt messages
        system_msg = (
            "Eres un analista experto en Recuperacion de Informacion y en Explicabilidad de sistemas de embeddings. "
            "Has sido formado en el MIT y tu estilo es tecnico, preciso y pedagogico. "
            "Tu tarea es analizar, de forma global y comparativa, los resultados de un sistema de busqueda que usa distintos "
            "metodos de chunking (fixed, sentence, adaptive). "
            "IMPORTANTE: No solo compares metricas. Explica pedagogicamente COMO FUNCIONA cada metodo, sus ventajas teoricas, "
            "limitaciones intrinsecas y por que el patron observado es consistente con la teoria. "
            "Reglas: No inventes informacion; no modifiques valores; si faltan datos, dilo; tono tecnico; concision; "
            "apoya con datos y snippets; indica cuando diferencias sean pequenas. "
            "FORMATO: Devuelve (a) resumen breve (max 4 frases) explicando tendencias y por que, "
            "(b) un JSON con estructura exacta: "
            "{\"summary\": \"\", \"per_method\": [{\"method\": \"\", \"reasoning\": \"\", \"evidence\": \"\", \"risks\": \"\"}], \"recommendation\": \"\"}"
        )

        user_msg = (
            "Analiza los siguientes resultados globales obtenidos tras una busqueda. Cada metodo de chunking incluye medidas numericas agregadas y fragmentos representativos. "
            "INSTRUCCIONES ESPECIFICAS: "
            "1) EXPLICA COMO FUNCIONA cada metodo teoricamente (fixed: tamano fijo + overlap; sentence: agrupa oraciones naturales; adaptive: similitud semantica entre oraciones). "
            "2) CONECTA la teoria con los resultados observados (ej: 'fixed tiende a chunks uniformes pero puede cortar conceptos'). "
            "3) USA las metricas para validar/refutar expectativas teoricas. "
            "4) CITA frases especificas de los snippets como evidencia. "
            "5) Si las diferencias son pequenas, explicalo y sugiere factores que podrian ser mas determinantes. "
            "Usa estilo tecnico y pedagogico (nivel MIT-PhD pero entendible). No generes texto fuera del material proporcionado.\n\n"
            f"QUERY:\n{payload['query_text']}\n\nRESULTADOS (JSON):\n{json.dumps(payload, ensure_ascii=False)}\n\n"
            "Cada metodo contiene: avg_cosine (similitud coseno media), coverage_ratio (% terminos query en chunks), "
            "idf_mean (peso IDF medio de terminos), avg_chunk_length (longitud promedio), examples (snippets con metricas).\n\n"
            "TASK DETALLADA: "
            "1) Tendencias globales y EXPLICACION TEORICA de cada metodo. "
            "2) Factores numericos que explican diferencias + conexion con principios de cada metodo. "
            "3) Frases de snippets que ilustran fortalezas/debilidades de cada enfoque. "
            "4) Riesgos inherentes a cada metodo (dilucion, perdida contexto, fronteras artificiales, etc.). "
            "5) Recomendacion fundamentada para este tipo de query y corpus.\n"
            "FORMATO: resumen tecnico (max 4 frases) + JSON valido con estructura exacta. Si falta info, usa 'informacion insuficiente'."
        )

        client = _get_openai_client()
        completion = client.chat.completions.create(
            model="gpt-4o-mini",
            temperature=0.3,
            max_tokens=1000,
            messages=[
                {"role": "system", "content": system_msg},
                {"role": "user", "content": user_msg},
            ],
        )
        content = completion.choices[0].message.content if completion and completion.choices else ""

        # Try to parse JSON from content
        summary_text = ""
        json_obj = None
        # Heuristic: find JSON block
        if content:
            try:
                # Direct parse
                json_obj = json.loads(content)
            except Exception:
                # Extract first {...} block
                start = content.find("{")
                end = content.rfind("}")
                if start != -1 and end != -1 and end > start:
                    try:
                        json_obj = json.loads(content[start:end+1])
                    except Exception:
                        json_obj = None

        if not json_obj:
            return (
                "⚠️ No se pudo generar la explicación en formato JSON válido.",
                content[:800] if content else "",
                pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]),
                "",
            )

        summary_text = str(json_obj.get("summary", ""))
        per_method = json_obj.get("per_method", []) or []
        recommendation = str(json_obj.get("recommendation", ""))

        table_rows = []
        for item in per_method:
            table_rows.append({
                "Method": item.get("method", ""),
                "Reasoning": item.get("reasoning", ""),
                "Evidence": item.get("evidence", ""),
                "Risks": item.get("risks", ""),
            })
        table_df = pd.DataFrame(table_rows, columns=["Method","Reasoning","Evidence","Risks"])

        return ("", summary_text, table_df, recommendation)

    except Exception as e:
        logger.error(f"Explainability error: {str(e)}")
        return (
            "❌ No se pudo generar la explicación. Verifica tu clave de API o la conexión.",
            "",
            pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]),
            "",
        )
    """
    Get current system status including collections info.
    
    Returns:
        Tuple of (status_message, status_dataframe)
    """
    try:
        # Get collection statistics
        status_data = []
        total_chunks = 0
        
        for method in settings.CHUNKING_METHODS:
            collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method}"
            try:
                # This would need a method to get collection info
                # For now, we'll show configured methods
                status_data.append({
                    "Collection": collection_name,
                    "Method": method,
                    "Status": "Available",
                    "Description": f"Collection for {method} chunking method"
                })
            except Exception:
                status_data.append({
                    "Collection": collection_name,
                    "Method": method,
                    "Status": "Not Available",
                    "Description": f"Collection for {method} chunking method (not created)"
                })
        
        status_df = pd.DataFrame(status_data)
        
        # System configuration info
        config_info = f"""
        📊 **System Configuration:**
        - Embedding Model: {settings.OPENAI_EMBEDDING_MODEL}
        - Embedding Dimensions: {settings.EMBEDDING_DIMENSIONS}
        - Qdrant URL: {settings.QDRANT_URL}
        - Top K Results: {settings.TOP_K_RESULTS}
        - Min Similarity Score: {settings.MIN_SIMILARITY_SCORE}
        - Available Chunking Methods: {', '.join(settings.CHUNKING_METHODS)}
        """
        
        return config_info, status_df
        
    except Exception as e:
        logger.error(f"Error getting system status: {str(e)}")
        return f"❌ Error: {str(e)}", pd.DataFrame()

# Create Gradio interface
with gr.Blocks(title="Sistema de Chunking y Búsqueda de Documentos", theme=gr.themes.Soft(), css="""
    .gradio-container {
        max-width: 1200px !important;
    }
    .dataframe table {
        table-layout: fixed !important;
        word-wrap: break-word !important;
    }
    .dataframe td {
        white-space: pre-wrap !important;
        vertical-align: top !important;
        padding: 8px !important;
    }
""") as demo:
    gr.Markdown("""
    # 📄 Sistema de Chunking y Búsqueda de Documentos
    
    Este sistema procesa documentos utilizando múltiples estrategias de chunking y permite búsqueda semántica usando embeddings de OpenAI almacenados en Qdrant.
    
    ## Métodos de Chunking Disponibles:
    - **Fixed**: Chunks de tamaño fijo con solapamiento
    - **Sentence**: Chunking basado en oraciones
    - **Adaptive**: Chunking adaptativo basado en la estructura del contenido
    """)
    
    with gr.Tab("📤 Procesamiento de Documentos"):
        gr.Markdown("### Cargar y procesar documentos")
        
        with gr.Row():
            with gr.Column():
                file_input = gr.File(
                    label="Cargar Documento", 
                    file_types=[".pdf", ".txt", ".docx"],
                    type="filepath"
                )
                
                chunking_methods = gr.CheckboxGroup(
                    choices=settings.CHUNKING_METHODS,
                    value=settings.CHUNKING_METHODS,  # Select all by default
                    label="Seleccionar Métodos de Chunking"
                )
                
                process_btn = gr.Button("🔄 Procesar Documento", variant="primary")
            
            with gr.Column():
                process_status = gr.Textbox(
                    label="Estado del Procesamiento",
                    placeholder="Carga un documento y haz clic en 'Procesar Documento'",
                    interactive=False
                )
        
        chunks_table = gr.Dataframe(
            label="Resumen de Chunks",
            headers=["Método de Chunking", "Número de Chunks", "Longitud Promedio de Chunk", "Chunk de Muestra"],
            interactive=False,
            wrap=True,
            column_widths=["15%", "15%", "15%", "55%"],
            max_height=400
        )
        
        process_btn.click(
            process_document,
            inputs=[file_input, chunking_methods],
            outputs=[process_status, chunks_table]
        )
    
    with gr.Tab("🔍 Búsqueda"):
        gr.Markdown("### Buscar contenido similar en todos los documentos procesados")
        
        with gr.Row():
            with gr.Column():
                search_input = gr.Textbox(
                    label="Consulta de Búsqueda",
                    placeholder="Ingresa tu consulta de búsqueda aquí...",
                    lines=2
                )
                
                top_k_input = gr.Slider(
                    minimum=1,
                    maximum=20,
                    value=settings.TOP_K_RESULTS,
                    step=1,
                    label="Número de Resultados"
                )
                
                search_btn = gr.Button("🔍 Buscar", variant="primary")
            
            with gr.Column():
                search_status = gr.Textbox(
                    label="Estado de la Búsqueda",
                    placeholder="Ingresa una consulta y haz clic en 'Buscar'",
                    interactive=False
                )
        
        results_table = gr.Dataframe(
            label="Resultados de Búsqueda",
            headers=["Rango", "Puntuación de Similitud", "Método de Chunking", "Vista Previa de Texto", "Texto Completo"],
            interactive=False,
            wrap=True,
            column_widths=["8%", "12%", "15%", "30%", "35%"],
            max_height=600
        )
        
        # Explainability panel
        gr.Markdown("### 🧠 Explicabilidad de los resultados")
        with gr.Row():
            with gr.Column(scale=1):
                explain_btn = gr.Button("🧠 Explicabilidad de los resultados", variant="secondary", interactive=False)
                explain_status = gr.Textbox(label="Estado", interactive=False)
            with gr.Column(scale=2):
                explain_summary = gr.Textbox(label="Resumen técnico", lines=4, interactive=False)
                per_method_table = gr.Dataframe(
                    label="Análisis por método",
                    headers=["Method", "Reasoning", "Evidence", "Risks"],
                    interactive=False,
                    wrap=True,
                    max_height=300
                )
                recommendation_text = gr.Textbox(label="Recomendación", lines=2, interactive=False)

        # Search action
        search_event = search_btn.click(
            search_chunks,
            inputs=[search_input, top_k_input],
            outputs=[search_status, results_table]
        )

        # After search: enable explain button and clear previous explanation
        search_event.then(
            lambda: gr.update(interactive=True),
            inputs=None,
            outputs=explain_btn
        ).then(
            lambda: ("", "", pd.DataFrame(columns=["Method","Reasoning","Evidence","Risks"]), ""),
            inputs=None,
            outputs=[explain_status, explain_summary, per_method_table, recommendation_text]
        )

        # Explain action
        explain_btn.click(
            explain_ranking,
            inputs=[search_input, results_table],
            outputs=[explain_status, explain_summary, per_method_table, recommendation_text]
        )
    
    with gr.Tab("⚙️ Estado del Sistema"):
        gr.Markdown("### Configuración y estado actual del sistema")
        
        status_btn = gr.Button("🔄 Actualizar Estado", variant="secondary")
        
        system_info = gr.Textbox(
            label="Información del Sistema",
            lines=10,
            interactive=False
        )
        
        collections_table = gr.Dataframe(
            label="Collections Status",
            headers=["Colección", "Método", "Estado", "Descripción"],
            interactive=False,
            wrap=True,
            column_widths=["25%", "15%", "15%", "45%"],
            max_height=300
        )
        
        # Load initial status
        demo.load(
            get_system_status,
            outputs=[system_info, collections_table]
        )
        
        status_btn.click(
            get_system_status,
            outputs=[system_info, collections_table]
        )
    
    # Data Management Tab
    with gr.Tab("🗄️ Gestión de Datos"):
        gr.Markdown("## Gestión de Base de Datos Qdrant")
        gr.Markdown("Gestiona tus colecciones de documentos y visualiza datos usando las herramientas nativas de Qdrant.")
        
        with gr.Row():
            with gr.Column():
                gr.Markdown("### 📊 Visualización")
                dashboard_btn = gr.Button("🌐 Obtener Acceso al Dashboard", variant="primary", size="lg")
                dashboard_output = gr.Textbox(
                    label="Información de Acceso al Dashboard de Qdrant",
                    placeholder="Haz clic en el botón para obtener la URL del dashboard y detalles de acceso",
                    lines=15,
                    interactive=False
                )
                
            with gr.Column():
                gr.Markdown("### 📋 Resumen de Colecciones")
                summary_btn = gr.Button("📊 Obtener Resumen de Colecciones", variant="secondary")
                summary_output = gr.Textbox(
                    label="Información de Colecciones",
                    lines=8,
                    interactive=False
                )
                summary_table = gr.Dataframe(
                    label="Detalles de Colecciones",
                    headers=["Nombre de Colección", "Cantidad de Puntos", "Cantidad de Vectores", "Estado", "Estado del Optimizador"],
                    interactive=False,
                    wrap=True,
                    max_height=200
                )
        
        gr.Markdown("### 🗑️ Gestión de Colecciones")
        with gr.Row():
            with gr.Column():
                gr.Markdown("**Eliminar Colección Específica:**")
                method_dropdown = gr.Dropdown(
                    choices=settings.CHUNKING_METHODS,
                    label="Seleccionar Método de Chunking",
                    value=settings.CHUNKING_METHODS[0] if settings.CHUNKING_METHODS else None
                )
                delete_specific_btn = gr.Button("🗑️ Eliminar Colección Seleccionada", variant="secondary")
                
            with gr.Column():
                gr.Markdown("**Eliminar Todas las Colecciones:**")
                gr.Markdown("⚠️ **Advertencia**: ¡Esto eliminará TODAS las colecciones permanentemente!")
                delete_all_btn = gr.Button("🗑️ Eliminar Todas las Colecciones", variant="stop")

        with gr.Row():
            with gr.Column():
                gr.Markdown("**Reiniciar Seguimiento de Documentos:**")
                gr.Markdown("🔄 Forzar limpieza de colecciones en la próxima carga de documento (útil cuando se reprocesa el mismo archivo)")
                reset_tracking_btn = gr.Button("🔄 Reiniciar Seguimiento de Documentos", variant="secondary")
        
        delete_output = gr.Textbox(
            label="Estado de Eliminación",
            lines=3,
            interactive=False
        )
        
        # Event handlers for Data Management
        dashboard_btn.click(
            get_qdrant_dashboard_url,
            outputs=dashboard_output
        )
        
        summary_btn.click(
            get_collections_summary,
            outputs=[summary_output, summary_table]
        )
        
        def _delete_by_method(method_name: str) -> str:
            try:
                if not method_name:
                    return "❌ Por favor selecciona un método"
                collection_name = f"{settings.QDRANT_COLLECTION_NAME}_{method_name}"
                return delete_collection(collection_name)
            except Exception as e:
                return f"❌ Error eliminando: {str(e)}"

        delete_specific_btn.click(
            _delete_by_method,
            inputs=method_dropdown,
            outputs=delete_output
        )
        
        delete_all_btn.click(
            delete_all_collections,
            outputs=delete_output
        )
        
        reset_tracking_btn.click(
            reset_document_tracking,
            outputs=delete_output
        )

if __name__ == "__main__":
    # Set up environment for deployment
    port = int(os.environ.get("PORT", 7860))
    
    print("🚀 Iniciando Sistema de Chunking y Búsqueda de Documentos...")
    
    # Initialize components with error handling
    init_success = initialize_components()
    
    if not init_success:
        print("⚠️  Advertencia: Algunos componentes fallaron al inicializar. Verifica la configuración.")
        print("📋 Asegúrate de tener:")
        print("   - Variable de entorno OPENAI_API_KEY configurada")
        print("   - Instancia de Qdrant ejecutándose (QDRANT_URL configurado)")
    else:
        print("✅ Todos los componentes inicializados exitosamente")
    
    print(f"📊 Configurado para {len(settings.CHUNKING_METHODS)} métodos de chunking")
    print(f"🔗 URL de Qdrant: {settings.QDRANT_URL}")
    print(f"🤖 Modelo de Embeddings: {settings.OPENAI_EMBEDDING_MODEL}")
    
    demo.launch(
        server_name="0.0.0.0",
        server_port=port,
        share=False  # Gradio Spaces will handle public access
    )