""" Capa 3 · RAG con ChromaDB ========================== Embeddings via NVIDIA NIM API (nvidia/nv-embedqa-e5-v5) — H100. NO usa SentenceTransformer local — todo por NVIDIA API. Colecciones: - banxico_historico → series históricas 2024-2026 - banxico_actual → indicadores actuales Banxico - worldbank → 22 indicadores World Bank - inegi → indicadores INEGI - ift → sector telecomunicaciones México Flujo: 1. Al arrancar: indexar JSONs del bucket → ChromaDB en /data/chroma_db/ 2. Embeddings via NVIDIA NIM — misma H100 que los LLMs 3. El orquestador llama search_chroma() para búsqueda semántica """ import json import os import hashlib import requests from datetime import datetime DATA_PATH = "/data" CHROMA_PATH = "/tmp/chroma_db" MODEL_NAME = "nvidia/nv-embedqa-e5-v5" NVIDIA_BASE = "https://integrate.api.nvidia.com/v1" EMBED_DIM = 1024 # dimensión de nv-embedqa-e5-v5 _chroma_client = None _collections = {} _index_status = {} def _get_nvidia_key() -> str: return os.getenv("NVIDIA_API_KEY", "") # ─── Embeddings via NVIDIA NIM ──────────────────────────────────────────────── # ─── Embedding strategy ────────────────────────────────────────────────────── # INDEXING: SentenceTransformer local (fast, ~30s for 22k docs, no API calls) # QUERYING: NVIDIA NIM (high quality semantic search, 1 call per query) _st_model = None def _get_local_model(): """Lazy load SentenceTransformer for indexing.""" global _st_model if _st_model is None: from sentence_transformers import SentenceTransformer _st_model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2") return _st_model def _get_embeddings(texts: list, input_type: str = "passage") -> list: """ Genera embeddings: - "passage" (indexado) → SentenceTransformer local (384d, rápido) - "query" (búsqueda) → NVIDIA NIM si disponible, sino local """ if input_type == "passage": # Indexado: siempre local para velocidad model = _get_local_model() return model.encode(texts, batch_size=128, show_progress_bar=False).tolist() # Búsqueda: NVIDIA NIM si hay API key api_key = _get_nvidia_key() if api_key: try: headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", } r = requests.post( f"{NVIDIA_BASE}/embeddings", headers=headers, json={ "model": MODEL_NAME, "input": texts, "input_type": "query", "encoding_format": "float", "truncate": "END", }, timeout=15, ) r.raise_for_status() data = sorted(r.json()["data"], key=lambda x: x["index"]) return [item["embedding"] for item in data] except Exception: pass # Fallback: local para búsqueda también model = _get_local_model() return model.encode(texts, show_progress_bar=False).tolist() # ─── ChromaDB client ────────────────────────────────────────────────────────── def _get_chroma_client(): global _chroma_client if _chroma_client is None: import chromadb os.makedirs(CHROMA_PATH, exist_ok=True) _chroma_client = chromadb.EphemeralClient() return _chroma_client def _get_collection(name: str): if name not in _collections: client = _get_chroma_client() _collections[name] = client.get_or_create_collection( name=name, metadata={"hnsw:space": "cosine"} ) return _collections[name] def _file_hash(path: str) -> str: try: with open(path, "rb") as f: return hashlib.md5(f.read()).hexdigest() except Exception: return "" def _needs_reindex(collection_name: str, file_path: str) -> bool: current_hash = _file_hash(file_path) last_hash = _index_status.get(collection_name, {}).get("hash", "") return current_hash != last_hash # ─── Indexadores por fuente ─────────────────────────────────────────────────── def index_banxico_historico(force: bool = False) -> dict: path = os.path.join(DATA_PATH, "banxico_historico.json") collection_name = "banxico_historico" if not force and not _needs_reindex(collection_name, path): return {"status": "skip", "reason": "Sin cambios desde último indexado"} try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) collection = _get_collection(collection_name) nombres = { "SF43718": "Tipo de cambio USD/MXN", "SF43783": "TIIE 28 días", "SF61745": "Tasa objetivo Banxico", "SP1": "Inflación IPC", } documents, metadatas, ids = [], [], [] for serie_id, registros in data.items(): if serie_id in ["fuente", "timestamp", "periodo"] or not isinstance(registros, list): continue nombre_serie = nombres.get(serie_id, serie_id) # Limitar a últimos 3000 registros para velocidad de indexado # Los datos más antiguos se sirven directo del JSON via _search_historico_by_year registros = registros[-3000:] if len(registros) > 3000 else registros for i, registro in enumerate(registros): fecha = registro.get("fecha", registro.get("date", "sin_fecha")) valor = registro.get("dato", registro.get("valor", registro.get("value", "N/A"))) texto = ( f"{nombre_serie} ({serie_id}) el {fecha}: {valor}. " f"Serie: {serie_id}. " f"Indicador Banxico de tipo {nombre_serie.lower()}." ) documents.append(texto) metadatas.append({ "serie": serie_id, "nombre": nombre_serie, "fecha": str(fecha), "valor": str(valor), "fuente": "Banxico SIE API", }) ids.append(f"{serie_id}_{i}") if not documents: return {"status": "error", "reason": "Sin registros en banxico_historico.json"} # Batch size 500 — SentenceTransformer local maneja bien lotes grandes batch_size = 500 total = 0 model = _get_local_model() for i in range(0, len(documents), batch_size): batch_docs = documents[i:i+batch_size] batch_meta = metadatas[i:i+batch_size] batch_ids = ids[i:i+batch_size] # Encodar todo el batch de una vez — más eficiente embeddings = model.encode( batch_docs, batch_size=128, show_progress_bar=False, convert_to_numpy=True ).tolist() collection.upsert( documents=batch_docs, embeddings=embeddings, metadatas=batch_meta, ids=batch_ids, ) total += len(batch_docs) _index_status[collection_name] = { "indexed": True, "docs": total, "hash": _file_hash(path), "ts": datetime.now().isoformat() } return {"status": "ok", "docs_indexados": total, "coleccion": collection_name} except FileNotFoundError: return {"status": "error", "reason": "banxico_historico.json no encontrado"} except Exception as e: return {"status": "error", "reason": str(e)} def index_banxico_actual(force: bool = False) -> dict: path = os.path.join(DATA_PATH, "banxico.json") collection_name = "banxico_actual" if not force and not _needs_reindex(collection_name, path): return {"status": "skip", "reason": "Sin cambios"} try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) collection = _get_collection(collection_name) ts = data.get("timestamp", data.get("actualizado", "N/A")) def extraer_indicadores(d, prefix=""): items = [] for k, v in d.items(): if k in ["fuente", "timestamp", "actualizado", "archivo"]: continue if isinstance(v, dict): valor = v.get("valor", v.get("valor_formateado", str(v))) fecha = v.get("fecha", ts) items.append((f"{prefix}{k}", str(valor), str(fecha))) items.extend(extraer_indicadores(v, prefix=f"{k}.")) elif not isinstance(v, list): items.append((f"{prefix}{k}", str(v), ts)) return items indicadores = extraer_indicadores(data) if not indicadores: return {"status": "error", "reason": "Sin indicadores en banxico.json"} documents, metadatas, ids = [], [], [] for nombre, valor, fecha in indicadores: texto = ( f"Indicador Banxico {nombre}: {valor} (fecha: {fecha}). " f"Dato actual del Banco de México." ) documents.append(texto) metadatas.append({"indicador": nombre, "valor": valor, "fecha": fecha, "fuente": "Banxico"}) ids.append(f"banxico_{nombre.replace('.','_')}") embeddings = _get_embeddings(documents, input_type="passage") collection.upsert(documents=documents, embeddings=embeddings, metadatas=metadatas, ids=ids) _index_status[collection_name] = { "indexed": True, "docs": len(documents), "hash": _file_hash(path), "ts": datetime.now().isoformat() } return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} except FileNotFoundError: return {"status": "error", "reason": "banxico.json no encontrado"} except Exception as e: return {"status": "error", "reason": str(e)} def index_worldbank(force: bool = False) -> dict: path = os.path.join(DATA_PATH, "worldbank.json") collection_name = "worldbank" if not force and not _needs_reindex(collection_name, path): return {"status": "skip", "reason": "Sin cambios"} try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) collection = _get_collection(collection_name) documents, metadatas, ids = [], [], [] for categoria, indicadores in data.items(): if categoria in ["fuente", "timestamp", "pais"] or not isinstance(indicadores, dict): continue for nombre, info in indicadores.items(): if isinstance(info, dict): valor = info.get("valor_formateado", info.get("valor", str(info))) anio = info.get("anio", info.get("year", "N/A")) unidad = info.get("unidad", "") else: valor = str(info) anio = "N/A" unidad = "" texto = ( f"World Bank México — {nombre} ({categoria}): {valor} {unidad} " f"(año: {anio}). Indicador del Banco Mundial para México." ) doc_id = f"wb_{categoria}_{nombre.replace(' ','_')[:40]}" documents.append(texto) metadatas.append({ "categoria": categoria, "indicador": nombre, "valor": str(valor), "anio": str(anio), "fuente": "World Bank" }) ids.append(doc_id) if not documents: return {"status": "error", "reason": "Sin datos en worldbank.json"} embeddings = _get_embeddings(documents, input_type="passage") collection.upsert(documents=documents, embeddings=embeddings, metadatas=metadatas, ids=ids) _index_status[collection_name] = { "indexed": True, "docs": len(documents), "hash": _file_hash(path), "ts": datetime.now().isoformat() } return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} except FileNotFoundError: return {"status": "error", "reason": "worldbank.json no encontrado"} except Exception as e: return {"status": "error", "reason": str(e)} def index_inegi(force: bool = False) -> dict: path = os.path.join(DATA_PATH, "inegi.json") collection_name = "inegi" if not force and not _needs_reindex(collection_name, path): return {"status": "skip", "reason": "Sin cambios"} try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) collection = _get_collection(collection_name) ts = data.get("timestamp", "N/A") documents, metadatas, ids = [], [], [] def flatten(d, prefix=""): for k, v in d.items(): if k in ["fuente", "timestamp", "detalle"]: continue if isinstance(v, dict): flatten(v, prefix=f"{k}.") elif not isinstance(v, list): texto = ( f"INEGI México — {prefix}{k}: {v} " f"(fecha extracción: {ts}). " f"Indicador del Instituto Nacional de Estadística." ) doc_id = f"inegi_{(prefix+k).replace('.','_')[:50]}" documents.append(texto) metadatas.append({ "indicador": f"{prefix}{k}", "valor": str(v), "fecha": ts, "fuente": "INEGI" }) ids.append(doc_id) flatten(data) if "detalle" in data and isinstance(data["detalle"], dict): flatten(data["detalle"], prefix="detalle.") if not documents: return {"status": "error", "reason": "Sin datos en inegi.json"} embeddings = _get_embeddings(documents, input_type="passage") collection.upsert(documents=documents, embeddings=embeddings, metadatas=metadatas, ids=ids) _index_status[collection_name] = { "indexed": True, "docs": len(documents), "hash": _file_hash(path), "ts": datetime.now().isoformat() } return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} except FileNotFoundError: return {"status": "error", "reason": "inegi.json no encontrado"} except Exception as e: return {"status": "error", "reason": str(e)} def index_ift(force: bool = False) -> dict: path = os.path.join(DATA_PATH, "ift.json") collection_name = "ift" if not force and not _needs_reindex(collection_name, path): return {"status": "skip", "reason": "Sin cambios"} try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) collection = _get_collection(collection_name) documents, metadatas, ids = [], [], [] periodo = data.get("metadatos", {}).get("periodo", "Q1 2025") for op in data.get("ingresos_q1_2025", {}).get("operadores", []): marca = op.get("marca", "") grupo = op.get("grupo", "") ingreso = op.get("ingreso_millones_mxn", "") pct = op.get("participacion_pct", "") texto = ( f"IFT Telecomunicaciones México — {marca} ({grupo}): " f"ingresos {ingreso} millones MXN, participación {pct}% del sector. " f"Período: {periodo}. Fuente: IFT datos oportunos." ) documents.append(texto) metadatas.append({ "categoria": "ingresos", "operador": marca, "valor": str(ingreso), "participacion": str(pct), "fuente": "IFT", "periodo": periodo }) ids.append(f"ift_ingreso_{marca.replace(' ','_')[:30]}") for op in data.get("participacion_mercado_movil", {}).get("operadores", []): operador = op.get("operador", "") lineas = op.get("lineas_millones", "") pct = op.get("participacion_pct", "") texto = ( f"IFT México mercado móvil — {operador}: " f"{lineas} millones de líneas, {pct}% participación de mercado. " f"Período: Q3 2024. Fuente: IFT." ) documents.append(texto) metadatas.append({ "categoria": "mercado_movil", "operador": operador, "valor": str(lineas), "participacion": str(pct), "fuente": "IFT", "periodo": "Q3 2024" }) ids.append(f"ift_movil_{operador.replace(' ','_')[:30]}") conn = data.get("indicadores_conectividad", {}) for k, v in conn.items(): if k in ["fuente", "periodo"] or isinstance(v, dict): continue texto = ( f"IFT México conectividad — {k}: {v}. " f"Indicador de telecomunicaciones México. Fuente: IFT/ENDUTIH 2024." ) documents.append(texto) metadatas.append({ "categoria": "conectividad", "indicador": k, "valor": str(v), "fuente": "IFT", "periodo": "2024" }) ids.append(f"ift_conn_{k[:40]}") cobertura = data.get("cobertura_5g", {}) texto_5g = ( f"IFT México cobertura 5G — Estado: {cobertura.get('estado_2024', '')}. " f"Operadores con 5G: {', '.join(cobertura.get('operadores_con_5g', []))}. " f"Ciudades principales 5G: {', '.join(cobertura.get('ciudades_principales_5g', []))}. " f"Penetración 5G estimada: {cobertura.get('penetracion_5g_estimada_pct', '')}%. " f"Cobertura 4G LTE: {cobertura.get('cobertura_4g_lte_pct_poblacion', '')}% población. " f"Fuente: IFT/GSMA Intelligence 2024." ) documents.append(texto_5g) metadatas.append({ "categoria": "cobertura_5g", "indicador": "5G_4G", "valor": str(cobertura.get("penetracion_5g_estimada_pct", "")), "fuente": "IFT/GSMA", "periodo": "2024" }) ids.append("ift_cobertura_5g") reg = data.get("contexto_regulatorio", {}) texto_reg = ( f"IFT regulación telecomunicaciones México — " f"Agente Económico Preponderante (AEP): {reg.get('agente_economico_preponderante', '')}. " f"Regulador: {reg.get('regulador', '')}. " f"{reg.get('nota_2025', '')} " f"{reg.get('sancion_telcel_2025', '')}" ) documents.append(texto_reg) metadatas.append({ "categoria": "regulacion", "indicador": "AEP", "valor": "America Movil", "fuente": "IFT", "periodo": "2025" }) ids.append("ift_regulacion") if not documents: return {"status": "error", "reason": "Sin datos en ift.json"} embeddings = _get_embeddings(documents, input_type="passage") collection.upsert(documents=documents, embeddings=embeddings, metadatas=metadatas, ids=ids) _index_status[collection_name] = { "indexed": True, "docs": len(documents), "hash": _file_hash(path), "ts": datetime.now().isoformat() } return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} except FileNotFoundError: return {"status": "error", "reason": "ift.json no encontrado — ejecutar data_extractor.py"} except Exception as e: return {"status": "error", "reason": str(e)} # ─── Indexado completo ──────────────────────────────────────────────────────── def index_all(force: bool = False) -> dict: results = { "banxico_historico": index_banxico_historico(force), "banxico_actual": index_banxico_actual(force), "worldbank": index_worldbank(force), "inegi": index_inegi(force), "ift": index_ift(force), } ok = sum(1 for r in results.values() if r.get("status") in ["ok", "skip"]) error = sum(1 for r in results.values() if r.get("status") == "error") return { "status": "ok" if error == 0 else "partial", "ok": ok, "errores": error, "detalle": results, "ts": datetime.now().isoformat(), } # ─── Búsqueda semántica ─────────────────────────────────────────────────────── def search_chroma(query: str, collection_name: str, n_results: int = 5, where: dict = None) -> list: try: collection = _get_collection(collection_name) if collection.count() == 0: return [{"error": f"Colección '{collection_name}' vacía — ejecutar index_all()"}] query_embedding = _get_embeddings([query], input_type="query") kwargs = { "query_embeddings": query_embedding, "n_results": min(n_results, collection.count()), "include": ["documents", "metadatas", "distances"], } if where: kwargs["where"] = where results = collection.query(**kwargs) output = [] for doc, meta, dist in zip( results["documents"][0], results["metadatas"][0], results["distances"][0], ): output.append({ "texto": doc, "metadata": meta, "distancia": round(dist, 4), "relevancia": round(1 - dist, 4), }) return output except Exception as e: return [{"error": str(e)}] def search_historico_semantico(query: str, serie: str = None, n_results: int = 50) -> dict: import re as _re years_mentioned = _re.findall(r'\b(202[0-9])\b', query) if years_mentioned: return _search_historico_by_year(query, years_mentioned, serie) q_lower = query.lower() if any(k in q_lower for k in ["evoluc", "tendencia", "trend", "durante", "todo", "año", "year", "máximo", "mínimo", "maximum", "minimum", "compara", "compare"]): n_results = max(n_results, 100) try: where = {"serie": serie} if serie else None results = search_chroma(query, "banxico_historico", n_results=n_results, where=where) if results and "error" in results[0]: results = search_chroma(query, "banxico_historico", n_results=n_results, where=None) except Exception: results = search_chroma(query, "banxico_historico", n_results=n_results, where=None) if results and "error" in results[0]: return {"error": results[0]["error"]} if serie: results = [r for r in results if r.get("metadata", {}).get("serie") == serie] registros = [] for r in results: meta = r.get("metadata", {}) registros.append({ "serie": meta.get("serie"), "nombre": meta.get("nombre"), "fecha": meta.get("fecha"), "valor": meta.get("valor"), "relevancia": r.get("relevancia"), }) return { "fuente": "ChromaDB · NVIDIA NIM embeddings", "modelo": MODEL_NAME, "query": query, "serie": serie or "todas", "resultados": registros, "total": len(registros), } def _search_historico_by_year(query: str, years: list, serie: str = None) -> dict: path = os.path.join(DATA_PATH, "banxico_historico.json") SERIES_NOMBRES = { "SF43718": "Tipo de cambio USD/MXN", "SF43783": "TIIE 28 días", "SF61745": "Tasa objetivo Banxico", "SP1": "Inflación IPC", } if not serie: q = query.lower() if any(k in q for k in ["tipo de cambio", "dólar", "dollar", "exchange", "peso", "sf43718"]): serie = "SF43718" elif any(k in q for k in ["tiie", "interbancaria", "sf43783"]): serie = "SF43783" elif any(k in q for k in ["tasa objetivo", "objetivo", "sf61745"]): serie = "SF61745" elif any(k in q for k in ["inflacion", "inflación", "ipc", "sp1"]): serie = "SP1" try: with open(path, "r", encoding="utf-8") as f: data = json.load(f) series_to_search = [serie] if serie else ["SF43718", "SF43783", "SF61745", "SP1"] registros = [] for sid in series_to_search: if sid not in data: continue for obs in data[sid]: fecha = obs.get("fecha", "") if any(year in fecha for year in years): try: registros.append({ "serie": sid, "nombre": SERIES_NOMBRES.get(sid, sid), "fecha": fecha, "valor": str(obs.get("dato", "")), "relevancia": 1.0, }) except Exception: pass def sort_key(r): f = r.get("fecha", "") try: for fmt in ["%d/%m/%Y", "%Y-%m-%d"]: try: return datetime.strptime(f, fmt) except Exception: pass except Exception: pass return f registros.sort(key=sort_key) valores = [] for r in registros: try: valores.append(float(r["valor"])) except Exception: pass stats = {} if valores: stats = { "minimo": round(min(valores), 4), "maximo": round(max(valores), 4), "promedio": round(sum(valores)/len(valores), 4), "primero": valores[0], "ultimo": valores[-1], } return { "fuente": "JSON directo · banxico_historico", "modelo": "búsqueda_directa", "query": query, "serie": serie or "todas", "años": years, "resultados": registros, "total": len(registros), "estadisticas": stats, "nota": f"Búsqueda directa por año(s): {', '.join(years)}", } except Exception as e: return {"error": str(e)} def search_all_sources(query: str, n_results: int = 3) -> dict: collections = ["banxico_actual", "banxico_historico", "worldbank", "inegi", "ift"] resultados = {} for col in collections: try: col_obj = _get_collection(col) if col_obj.count() == 0: resultados[col] = {"status": "vacía"} continue results = search_chroma(query, col, n_results=n_results) if results and "error" not in results[0]: resultados[col] = { "status": "ok", "resultados": [ {"texto": r["texto"][:200], "metadata": r["metadata"], "relevancia": r["relevancia"]} for r in results ] } else: resultados[col] = {"status": "error", "detalle": results} except Exception as e: resultados[col] = {"status": "error", "detalle": str(e)} return { "fuente": "ChromaDB · NVIDIA NIM embeddings · búsqueda multi-colección", "modelo": MODEL_NAME, "query": query, "fuentes": resultados, } # ─── Estado del índice ──────────────────────────────────────────────────────── def get_index_status() -> dict: status = {} for col_name in ["banxico_historico", "banxico_actual", "worldbank", "inegi", "ift"]: try: col = _get_collection(col_name) count = col.count() except Exception: count = 0 idx = _index_status.get(col_name, {}) status[col_name] = { "indexado": idx.get("indexed", False), "documentos": count, "ultimo_ts": idx.get("ts", "nunca"), "hash": idx.get("hash", "—"), } return status def init_chroma(force: bool = False) -> dict: """Punto de entrada único para inicializar ChromaDB al arrancar la app.""" return index_all(force=force) if __name__ == "__main__": print("Inicializando ChromaDB con NVIDIA NIM embeddings...") result = init_chroma(force=True) print(json.dumps(result, indent=2, ensure_ascii=False)) print("\nPrueba búsqueda IFT:") r = search_chroma("operadores 5G México", "ift", n_results=3) print(json.dumps(r, indent=2, ensure_ascii=False)) def reset_collections() -> dict: """ Borra físicamente /data/chroma_db/ y limpia cache en memoria. Necesario cuando cambia el modelo de embeddings (dimensión 384→1024). """ global _collections, _chroma_client import shutil # 1. Cerrar y destruir cliente ChromaDB en memoria try: if _chroma_client is not None: del _chroma_client except Exception: pass _chroma_client = None _collections = {} _index_status.clear() # 2. Borrar archivos físicos en disco if os.path.exists(CHROMA_PATH): try: shutil.rmtree(CHROMA_PATH) except Exception as e: return {"status": "error", "reason": f"No se pudo borrar {CHROMA_PATH}: {e}"} # 3. Recrear carpeta vacía con permisos correctos os.makedirs(CHROMA_PATH, exist_ok=True) os.chmod(CHROMA_PATH, 0o777) # 4. Crear cliente nuevo desde cero import chromadb _chroma_client = chromadb.EphemeralClient() return { "status": "ok", "deleted": CHROMA_PATH, "mensaje": "ChromaDB reseteado completamente. Cliente recreado. Listo para re-indexar.", } def init_chroma_fresh() -> dict: """ Borra físicamente ChromaDB y re-indexa desde cero con NVIDIA NIM embeddings. Usar cuando se cambia el modelo de embeddings (ej: 384→1024 dimensiones). """ reset_result = reset_collections() if reset_result.get("status") == "error": return {"status": "error", "reset": reset_result} index_result = index_all(force=True) return { "status": "ok" if index_result.get("errores", 1) == 0 else "partial", "reset": reset_result, "index": index_result, }