Spaces:
Sleeping
Sleeping
Download modules/rag_chroma.py from Fercor95mx/mx-data-api: direct link, hf CLI and curl.
- Browser
- Download file 31.8 kB
-
https://huggingface.co/spaces/Fercor95mx/mx-data-api/resolve/main/modules/rag_chroma.py
- Command line
-
hf download hf://spaces/Fercor95mx/mx-data-api/modules/rag_chroma.py
-
curl -L -o rag_chroma.py https://huggingface.co/spaces/Fercor95mx/mx-data-api/resolve/main/modules/rag_chroma.py
31.8 kB
| """ | |
| Capa 3 · RAG con ChromaDB | |
| ========================== | |
| Embeddings via NVIDIA NIM API (nvidia/nv-embedqa-e5-v5) — H100. | |
| NO usa SentenceTransformer local — todo por NVIDIA API. | |
| Colecciones: | |
| - banxico_historico → series históricas 2024-2026 | |
| - banxico_actual → indicadores actuales Banxico | |
| - worldbank → 22 indicadores World Bank | |
| - inegi → indicadores INEGI | |
| - ift → sector telecomunicaciones México | |
| Flujo: | |
| 1. Al arrancar: indexar JSONs del bucket → ChromaDB en /data/chroma_db/ | |
| 2. Embeddings via NVIDIA NIM — misma H100 que los LLMs | |
| 3. El orquestador llama search_chroma() para búsqueda semántica | |
| """ | |
| import json | |
| import os | |
| import hashlib | |
| import requests | |
| from datetime import datetime | |
| DATA_PATH = "/data" | |
| CHROMA_PATH = "/tmp/chroma_db" | |
| MODEL_NAME = "nvidia/nv-embedqa-e5-v5" | |
| NVIDIA_BASE = "https://integrate.api.nvidia.com/v1" | |
| EMBED_DIM = 1024 # dimensión de nv-embedqa-e5-v5 | |
| _chroma_client = None | |
| _collections = {} | |
| _index_status = {} | |
| def _get_nvidia_key() -> str: | |
| return os.getenv("NVIDIA_API_KEY", "") | |
| # ─── Embeddings via NVIDIA NIM ──────────────────────────────────────────────── | |
| # ─── Embedding strategy ────────────────────────────────────────────────────── | |
| # INDEXING: SentenceTransformer local (fast, ~30s for 22k docs, no API calls) | |
| # QUERYING: NVIDIA NIM (high quality semantic search, 1 call per query) | |
| _st_model = None | |
| def _get_local_model(): | |
| """Lazy load SentenceTransformer for indexing.""" | |
| global _st_model | |
| if _st_model is None: | |
| from sentence_transformers import SentenceTransformer | |
| _st_model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2") | |
| return _st_model | |
| def _get_embeddings(texts: list, input_type: str = "passage") -> list: | |
| """ | |
| Genera embeddings: | |
| - "passage" (indexado) → SentenceTransformer local (384d, rápido) | |
| - "query" (búsqueda) → NVIDIA NIM si disponible, sino local | |
| """ | |
| if input_type == "passage": | |
| # Indexado: siempre local para velocidad | |
| model = _get_local_model() | |
| return model.encode(texts, batch_size=128, show_progress_bar=False).tolist() | |
| # Búsqueda: NVIDIA NIM si hay API key | |
| api_key = _get_nvidia_key() | |
| if api_key: | |
| try: | |
| headers = { | |
| "Authorization": f"Bearer {api_key}", | |
| "Content-Type": "application/json", | |
| } | |
| r = requests.post( | |
| f"{NVIDIA_BASE}/embeddings", | |
| headers=headers, | |
| json={ | |
| "model": MODEL_NAME, | |
| "input": texts, | |
| "input_type": "query", | |
| "encoding_format": "float", | |
| "truncate": "END", | |
| }, | |
| timeout=15, | |
| ) | |
| r.raise_for_status() | |
| data = sorted(r.json()["data"], key=lambda x: x["index"]) | |
| return [item["embedding"] for item in data] | |
| except Exception: | |
| pass | |
| # Fallback: local para búsqueda también | |
| model = _get_local_model() | |
| return model.encode(texts, show_progress_bar=False).tolist() | |
| # ─── ChromaDB client ────────────────────────────────────────────────────────── | |
| def _get_chroma_client(): | |
| global _chroma_client | |
| if _chroma_client is None: | |
| import chromadb | |
| os.makedirs(CHROMA_PATH, exist_ok=True) | |
| _chroma_client = chromadb.EphemeralClient() | |
| return _chroma_client | |
| def _get_collection(name: str): | |
| if name not in _collections: | |
| client = _get_chroma_client() | |
| _collections[name] = client.get_or_create_collection( | |
| name=name, | |
| metadata={"hnsw:space": "cosine"} | |
| ) | |
| return _collections[name] | |
| def _file_hash(path: str) -> str: | |
| try: | |
| with open(path, "rb") as f: | |
| return hashlib.md5(f.read()).hexdigest() | |
| except Exception: | |
| return "" | |
| def _needs_reindex(collection_name: str, file_path: str) -> bool: | |
| current_hash = _file_hash(file_path) | |
| last_hash = _index_status.get(collection_name, {}).get("hash", "") | |
| return current_hash != last_hash | |
| # ─── Indexadores por fuente ─────────────────────────────────────────────────── | |
| def index_banxico_historico(force: bool = False) -> dict: | |
| path = os.path.join(DATA_PATH, "banxico_historico.json") | |
| collection_name = "banxico_historico" | |
| if not force and not _needs_reindex(collection_name, path): | |
| return {"status": "skip", "reason": "Sin cambios desde último indexado"} | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| collection = _get_collection(collection_name) | |
| nombres = { | |
| "SF43718": "Tipo de cambio USD/MXN", | |
| "SF43783": "TIIE 28 días", | |
| "SF61745": "Tasa objetivo Banxico", | |
| "SP1": "Inflación IPC", | |
| } | |
| documents, metadatas, ids = [], [], [] | |
| for serie_id, registros in data.items(): | |
| if serie_id in ["fuente", "timestamp", "periodo"] or not isinstance(registros, list): | |
| continue | |
| nombre_serie = nombres.get(serie_id, serie_id) | |
| # Limitar a últimos 3000 registros para velocidad de indexado | |
| # Los datos más antiguos se sirven directo del JSON via _search_historico_by_year | |
| registros = registros[-3000:] if len(registros) > 3000 else registros | |
| for i, registro in enumerate(registros): | |
| fecha = registro.get("fecha", registro.get("date", "sin_fecha")) | |
| valor = registro.get("dato", registro.get("valor", registro.get("value", "N/A"))) | |
| texto = ( | |
| f"{nombre_serie} ({serie_id}) el {fecha}: {valor}. " | |
| f"Serie: {serie_id}. " | |
| f"Indicador Banxico de tipo {nombre_serie.lower()}." | |
| ) | |
| documents.append(texto) | |
| metadatas.append({ | |
| "serie": serie_id, | |
| "nombre": nombre_serie, | |
| "fecha": str(fecha), | |
| "valor": str(valor), | |
| "fuente": "Banxico SIE API", | |
| }) | |
| ids.append(f"{serie_id}_{i}") | |
| if not documents: | |
| return {"status": "error", "reason": "Sin registros en banxico_historico.json"} | |
| # Batch size 500 — SentenceTransformer local maneja bien lotes grandes | |
| batch_size = 500 | |
| total = 0 | |
| model = _get_local_model() | |
| for i in range(0, len(documents), batch_size): | |
| batch_docs = documents[i:i+batch_size] | |
| batch_meta = metadatas[i:i+batch_size] | |
| batch_ids = ids[i:i+batch_size] | |
| # Encodar todo el batch de una vez — más eficiente | |
| embeddings = model.encode( | |
| batch_docs, | |
| batch_size=128, | |
| show_progress_bar=False, | |
| convert_to_numpy=True | |
| ).tolist() | |
| collection.upsert( | |
| documents=batch_docs, | |
| embeddings=embeddings, | |
| metadatas=batch_meta, | |
| ids=batch_ids, | |
| ) | |
| total += len(batch_docs) | |
| _index_status[collection_name] = { | |
| "indexed": True, "docs": total, | |
| "hash": _file_hash(path), "ts": datetime.now().isoformat() | |
| } | |
| return {"status": "ok", "docs_indexados": total, "coleccion": collection_name} | |
| except FileNotFoundError: | |
| return {"status": "error", "reason": "banxico_historico.json no encontrado"} | |
| except Exception as e: | |
| return {"status": "error", "reason": str(e)} | |
| def index_banxico_actual(force: bool = False) -> dict: | |
| path = os.path.join(DATA_PATH, "banxico.json") | |
| collection_name = "banxico_actual" | |
| if not force and not _needs_reindex(collection_name, path): | |
| return {"status": "skip", "reason": "Sin cambios"} | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| collection = _get_collection(collection_name) | |
| ts = data.get("timestamp", data.get("actualizado", "N/A")) | |
| def extraer_indicadores(d, prefix=""): | |
| items = [] | |
| for k, v in d.items(): | |
| if k in ["fuente", "timestamp", "actualizado", "archivo"]: | |
| continue | |
| if isinstance(v, dict): | |
| valor = v.get("valor", v.get("valor_formateado", str(v))) | |
| fecha = v.get("fecha", ts) | |
| items.append((f"{prefix}{k}", str(valor), str(fecha))) | |
| items.extend(extraer_indicadores(v, prefix=f"{k}.")) | |
| elif not isinstance(v, list): | |
| items.append((f"{prefix}{k}", str(v), ts)) | |
| return items | |
| indicadores = extraer_indicadores(data) | |
| if not indicadores: | |
| return {"status": "error", "reason": "Sin indicadores en banxico.json"} | |
| documents, metadatas, ids = [], [], [] | |
| for nombre, valor, fecha in indicadores: | |
| texto = ( | |
| f"Indicador Banxico {nombre}: {valor} (fecha: {fecha}). " | |
| f"Dato actual del Banco de México." | |
| ) | |
| documents.append(texto) | |
| metadatas.append({"indicador": nombre, "valor": valor, "fecha": fecha, "fuente": "Banxico"}) | |
| ids.append(f"banxico_{nombre.replace('.','_')}") | |
| embeddings = _get_embeddings(documents, input_type="passage") | |
| collection.upsert(documents=documents, embeddings=embeddings, | |
| metadatas=metadatas, ids=ids) | |
| _index_status[collection_name] = { | |
| "indexed": True, "docs": len(documents), | |
| "hash": _file_hash(path), "ts": datetime.now().isoformat() | |
| } | |
| return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} | |
| except FileNotFoundError: | |
| return {"status": "error", "reason": "banxico.json no encontrado"} | |
| except Exception as e: | |
| return {"status": "error", "reason": str(e)} | |
| def index_worldbank(force: bool = False) -> dict: | |
| path = os.path.join(DATA_PATH, "worldbank.json") | |
| collection_name = "worldbank" | |
| if not force and not _needs_reindex(collection_name, path): | |
| return {"status": "skip", "reason": "Sin cambios"} | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| collection = _get_collection(collection_name) | |
| documents, metadatas, ids = [], [], [] | |
| for categoria, indicadores in data.items(): | |
| if categoria in ["fuente", "timestamp", "pais"] or not isinstance(indicadores, dict): | |
| continue | |
| for nombre, info in indicadores.items(): | |
| if isinstance(info, dict): | |
| valor = info.get("valor_formateado", info.get("valor", str(info))) | |
| anio = info.get("anio", info.get("year", "N/A")) | |
| unidad = info.get("unidad", "") | |
| else: | |
| valor = str(info) | |
| anio = "N/A" | |
| unidad = "" | |
| texto = ( | |
| f"World Bank México — {nombre} ({categoria}): {valor} {unidad} " | |
| f"(año: {anio}). Indicador del Banco Mundial para México." | |
| ) | |
| doc_id = f"wb_{categoria}_{nombre.replace(' ','_')[:40]}" | |
| documents.append(texto) | |
| metadatas.append({ | |
| "categoria": categoria, "indicador": nombre, | |
| "valor": str(valor), "anio": str(anio), "fuente": "World Bank" | |
| }) | |
| ids.append(doc_id) | |
| if not documents: | |
| return {"status": "error", "reason": "Sin datos en worldbank.json"} | |
| embeddings = _get_embeddings(documents, input_type="passage") | |
| collection.upsert(documents=documents, embeddings=embeddings, | |
| metadatas=metadatas, ids=ids) | |
| _index_status[collection_name] = { | |
| "indexed": True, "docs": len(documents), | |
| "hash": _file_hash(path), "ts": datetime.now().isoformat() | |
| } | |
| return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} | |
| except FileNotFoundError: | |
| return {"status": "error", "reason": "worldbank.json no encontrado"} | |
| except Exception as e: | |
| return {"status": "error", "reason": str(e)} | |
| def index_inegi(force: bool = False) -> dict: | |
| path = os.path.join(DATA_PATH, "inegi.json") | |
| collection_name = "inegi" | |
| if not force and not _needs_reindex(collection_name, path): | |
| return {"status": "skip", "reason": "Sin cambios"} | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| collection = _get_collection(collection_name) | |
| ts = data.get("timestamp", "N/A") | |
| documents, metadatas, ids = [], [], [] | |
| def flatten(d, prefix=""): | |
| for k, v in d.items(): | |
| if k in ["fuente", "timestamp", "detalle"]: | |
| continue | |
| if isinstance(v, dict): | |
| flatten(v, prefix=f"{k}.") | |
| elif not isinstance(v, list): | |
| texto = ( | |
| f"INEGI México — {prefix}{k}: {v} " | |
| f"(fecha extracción: {ts}). " | |
| f"Indicador del Instituto Nacional de Estadística." | |
| ) | |
| doc_id = f"inegi_{(prefix+k).replace('.','_')[:50]}" | |
| documents.append(texto) | |
| metadatas.append({ | |
| "indicador": f"{prefix}{k}", "valor": str(v), | |
| "fecha": ts, "fuente": "INEGI" | |
| }) | |
| ids.append(doc_id) | |
| flatten(data) | |
| if "detalle" in data and isinstance(data["detalle"], dict): | |
| flatten(data["detalle"], prefix="detalle.") | |
| if not documents: | |
| return {"status": "error", "reason": "Sin datos en inegi.json"} | |
| embeddings = _get_embeddings(documents, input_type="passage") | |
| collection.upsert(documents=documents, embeddings=embeddings, | |
| metadatas=metadatas, ids=ids) | |
| _index_status[collection_name] = { | |
| "indexed": True, "docs": len(documents), | |
| "hash": _file_hash(path), "ts": datetime.now().isoformat() | |
| } | |
| return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} | |
| except FileNotFoundError: | |
| return {"status": "error", "reason": "inegi.json no encontrado"} | |
| except Exception as e: | |
| return {"status": "error", "reason": str(e)} | |
| def index_ift(force: bool = False) -> dict: | |
| path = os.path.join(DATA_PATH, "ift.json") | |
| collection_name = "ift" | |
| if not force and not _needs_reindex(collection_name, path): | |
| return {"status": "skip", "reason": "Sin cambios"} | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| collection = _get_collection(collection_name) | |
| documents, metadatas, ids = [], [], [] | |
| periodo = data.get("metadatos", {}).get("periodo", "Q1 2025") | |
| for op in data.get("ingresos_q1_2025", {}).get("operadores", []): | |
| marca = op.get("marca", "") | |
| grupo = op.get("grupo", "") | |
| ingreso = op.get("ingreso_millones_mxn", "") | |
| pct = op.get("participacion_pct", "") | |
| texto = ( | |
| f"IFT Telecomunicaciones México — {marca} ({grupo}): " | |
| f"ingresos {ingreso} millones MXN, participación {pct}% del sector. " | |
| f"Período: {periodo}. Fuente: IFT datos oportunos." | |
| ) | |
| documents.append(texto) | |
| metadatas.append({ | |
| "categoria": "ingresos", "operador": marca, | |
| "valor": str(ingreso), "participacion": str(pct), | |
| "fuente": "IFT", "periodo": periodo | |
| }) | |
| ids.append(f"ift_ingreso_{marca.replace(' ','_')[:30]}") | |
| for op in data.get("participacion_mercado_movil", {}).get("operadores", []): | |
| operador = op.get("operador", "") | |
| lineas = op.get("lineas_millones", "") | |
| pct = op.get("participacion_pct", "") | |
| texto = ( | |
| f"IFT México mercado móvil — {operador}: " | |
| f"{lineas} millones de líneas, {pct}% participación de mercado. " | |
| f"Período: Q3 2024. Fuente: IFT." | |
| ) | |
| documents.append(texto) | |
| metadatas.append({ | |
| "categoria": "mercado_movil", "operador": operador, | |
| "valor": str(lineas), "participacion": str(pct), | |
| "fuente": "IFT", "periodo": "Q3 2024" | |
| }) | |
| ids.append(f"ift_movil_{operador.replace(' ','_')[:30]}") | |
| conn = data.get("indicadores_conectividad", {}) | |
| for k, v in conn.items(): | |
| if k in ["fuente", "periodo"] or isinstance(v, dict): | |
| continue | |
| texto = ( | |
| f"IFT México conectividad — {k}: {v}. " | |
| f"Indicador de telecomunicaciones México. Fuente: IFT/ENDUTIH 2024." | |
| ) | |
| documents.append(texto) | |
| metadatas.append({ | |
| "categoria": "conectividad", "indicador": k, | |
| "valor": str(v), "fuente": "IFT", "periodo": "2024" | |
| }) | |
| ids.append(f"ift_conn_{k[:40]}") | |
| cobertura = data.get("cobertura_5g", {}) | |
| texto_5g = ( | |
| f"IFT México cobertura 5G — Estado: {cobertura.get('estado_2024', '')}. " | |
| f"Operadores con 5G: {', '.join(cobertura.get('operadores_con_5g', []))}. " | |
| f"Ciudades principales 5G: {', '.join(cobertura.get('ciudades_principales_5g', []))}. " | |
| f"Penetración 5G estimada: {cobertura.get('penetracion_5g_estimada_pct', '')}%. " | |
| f"Cobertura 4G LTE: {cobertura.get('cobertura_4g_lte_pct_poblacion', '')}% población. " | |
| f"Fuente: IFT/GSMA Intelligence 2024." | |
| ) | |
| documents.append(texto_5g) | |
| metadatas.append({ | |
| "categoria": "cobertura_5g", "indicador": "5G_4G", | |
| "valor": str(cobertura.get("penetracion_5g_estimada_pct", "")), | |
| "fuente": "IFT/GSMA", "periodo": "2024" | |
| }) | |
| ids.append("ift_cobertura_5g") | |
| reg = data.get("contexto_regulatorio", {}) | |
| texto_reg = ( | |
| f"IFT regulación telecomunicaciones México — " | |
| f"Agente Económico Preponderante (AEP): {reg.get('agente_economico_preponderante', '')}. " | |
| f"Regulador: {reg.get('regulador', '')}. " | |
| f"{reg.get('nota_2025', '')} " | |
| f"{reg.get('sancion_telcel_2025', '')}" | |
| ) | |
| documents.append(texto_reg) | |
| metadatas.append({ | |
| "categoria": "regulacion", "indicador": "AEP", | |
| "valor": "America Movil", "fuente": "IFT", "periodo": "2025" | |
| }) | |
| ids.append("ift_regulacion") | |
| if not documents: | |
| return {"status": "error", "reason": "Sin datos en ift.json"} | |
| embeddings = _get_embeddings(documents, input_type="passage") | |
| collection.upsert(documents=documents, embeddings=embeddings, | |
| metadatas=metadatas, ids=ids) | |
| _index_status[collection_name] = { | |
| "indexed": True, "docs": len(documents), | |
| "hash": _file_hash(path), "ts": datetime.now().isoformat() | |
| } | |
| return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name} | |
| except FileNotFoundError: | |
| return {"status": "error", "reason": "ift.json no encontrado — ejecutar data_extractor.py"} | |
| except Exception as e: | |
| return {"status": "error", "reason": str(e)} | |
| # ─── Indexado completo ──────────────────────────────────────────────────────── | |
| def index_all(force: bool = False) -> dict: | |
| results = { | |
| "banxico_historico": index_banxico_historico(force), | |
| "banxico_actual": index_banxico_actual(force), | |
| "worldbank": index_worldbank(force), | |
| "inegi": index_inegi(force), | |
| "ift": index_ift(force), | |
| } | |
| ok = sum(1 for r in results.values() if r.get("status") in ["ok", "skip"]) | |
| error = sum(1 for r in results.values() if r.get("status") == "error") | |
| return { | |
| "status": "ok" if error == 0 else "partial", | |
| "ok": ok, | |
| "errores": error, | |
| "detalle": results, | |
| "ts": datetime.now().isoformat(), | |
| } | |
| # ─── Búsqueda semántica ─────────────────────────────────────────────────────── | |
| def search_chroma(query: str, collection_name: str, n_results: int = 5, | |
| where: dict = None) -> list: | |
| try: | |
| collection = _get_collection(collection_name) | |
| if collection.count() == 0: | |
| return [{"error": f"Colección '{collection_name}' vacía — ejecutar index_all()"}] | |
| query_embedding = _get_embeddings([query], input_type="query") | |
| kwargs = { | |
| "query_embeddings": query_embedding, | |
| "n_results": min(n_results, collection.count()), | |
| "include": ["documents", "metadatas", "distances"], | |
| } | |
| if where: | |
| kwargs["where"] = where | |
| results = collection.query(**kwargs) | |
| output = [] | |
| for doc, meta, dist in zip( | |
| results["documents"][0], | |
| results["metadatas"][0], | |
| results["distances"][0], | |
| ): | |
| output.append({ | |
| "texto": doc, | |
| "metadata": meta, | |
| "distancia": round(dist, 4), | |
| "relevancia": round(1 - dist, 4), | |
| }) | |
| return output | |
| except Exception as e: | |
| return [{"error": str(e)}] | |
| def search_historico_semantico(query: str, serie: str = None, | |
| n_results: int = 50) -> dict: | |
| import re as _re | |
| years_mentioned = _re.findall(r'\b(202[0-9])\b', query) | |
| if years_mentioned: | |
| return _search_historico_by_year(query, years_mentioned, serie) | |
| q_lower = query.lower() | |
| if any(k in q_lower for k in ["evoluc", "tendencia", "trend", "durante", "todo", "año", "year", | |
| "máximo", "mínimo", "maximum", "minimum", "compara", "compare"]): | |
| n_results = max(n_results, 100) | |
| try: | |
| where = {"serie": serie} if serie else None | |
| results = search_chroma(query, "banxico_historico", n_results=n_results, where=where) | |
| if results and "error" in results[0]: | |
| results = search_chroma(query, "banxico_historico", n_results=n_results, where=None) | |
| except Exception: | |
| results = search_chroma(query, "banxico_historico", n_results=n_results, where=None) | |
| if results and "error" in results[0]: | |
| return {"error": results[0]["error"]} | |
| if serie: | |
| results = [r for r in results if r.get("metadata", {}).get("serie") == serie] | |
| registros = [] | |
| for r in results: | |
| meta = r.get("metadata", {}) | |
| registros.append({ | |
| "serie": meta.get("serie"), | |
| "nombre": meta.get("nombre"), | |
| "fecha": meta.get("fecha"), | |
| "valor": meta.get("valor"), | |
| "relevancia": r.get("relevancia"), | |
| }) | |
| return { | |
| "fuente": "ChromaDB · NVIDIA NIM embeddings", | |
| "modelo": MODEL_NAME, | |
| "query": query, | |
| "serie": serie or "todas", | |
| "resultados": registros, | |
| "total": len(registros), | |
| } | |
| def _search_historico_by_year(query: str, years: list, serie: str = None) -> dict: | |
| path = os.path.join(DATA_PATH, "banxico_historico.json") | |
| SERIES_NOMBRES = { | |
| "SF43718": "Tipo de cambio USD/MXN", | |
| "SF43783": "TIIE 28 días", | |
| "SF61745": "Tasa objetivo Banxico", | |
| "SP1": "Inflación IPC", | |
| } | |
| if not serie: | |
| q = query.lower() | |
| if any(k in q for k in ["tipo de cambio", "dólar", "dollar", "exchange", "peso", "sf43718"]): | |
| serie = "SF43718" | |
| elif any(k in q for k in ["tiie", "interbancaria", "sf43783"]): | |
| serie = "SF43783" | |
| elif any(k in q for k in ["tasa objetivo", "objetivo", "sf61745"]): | |
| serie = "SF61745" | |
| elif any(k in q for k in ["inflacion", "inflación", "ipc", "sp1"]): | |
| serie = "SP1" | |
| try: | |
| with open(path, "r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| series_to_search = [serie] if serie else ["SF43718", "SF43783", "SF61745", "SP1"] | |
| registros = [] | |
| for sid in series_to_search: | |
| if sid not in data: | |
| continue | |
| for obs in data[sid]: | |
| fecha = obs.get("fecha", "") | |
| if any(year in fecha for year in years): | |
| try: | |
| registros.append({ | |
| "serie": sid, | |
| "nombre": SERIES_NOMBRES.get(sid, sid), | |
| "fecha": fecha, | |
| "valor": str(obs.get("dato", "")), | |
| "relevancia": 1.0, | |
| }) | |
| except Exception: | |
| pass | |
| def sort_key(r): | |
| f = r.get("fecha", "") | |
| try: | |
| for fmt in ["%d/%m/%Y", "%Y-%m-%d"]: | |
| try: | |
| return datetime.strptime(f, fmt) | |
| except Exception: | |
| pass | |
| except Exception: | |
| pass | |
| return f | |
| registros.sort(key=sort_key) | |
| valores = [] | |
| for r in registros: | |
| try: | |
| valores.append(float(r["valor"])) | |
| except Exception: | |
| pass | |
| stats = {} | |
| if valores: | |
| stats = { | |
| "minimo": round(min(valores), 4), | |
| "maximo": round(max(valores), 4), | |
| "promedio": round(sum(valores)/len(valores), 4), | |
| "primero": valores[0], | |
| "ultimo": valores[-1], | |
| } | |
| return { | |
| "fuente": "JSON directo · banxico_historico", | |
| "modelo": "búsqueda_directa", | |
| "query": query, | |
| "serie": serie or "todas", | |
| "años": years, | |
| "resultados": registros, | |
| "total": len(registros), | |
| "estadisticas": stats, | |
| "nota": f"Búsqueda directa por año(s): {', '.join(years)}", | |
| } | |
| except Exception as e: | |
| return {"error": str(e)} | |
| def search_all_sources(query: str, n_results: int = 3) -> dict: | |
| collections = ["banxico_actual", "banxico_historico", "worldbank", "inegi", "ift"] | |
| resultados = {} | |
| for col in collections: | |
| try: | |
| col_obj = _get_collection(col) | |
| if col_obj.count() == 0: | |
| resultados[col] = {"status": "vacía"} | |
| continue | |
| results = search_chroma(query, col, n_results=n_results) | |
| if results and "error" not in results[0]: | |
| resultados[col] = { | |
| "status": "ok", | |
| "resultados": [ | |
| {"texto": r["texto"][:200], "metadata": r["metadata"], "relevancia": r["relevancia"]} | |
| for r in results | |
| ] | |
| } | |
| else: | |
| resultados[col] = {"status": "error", "detalle": results} | |
| except Exception as e: | |
| resultados[col] = {"status": "error", "detalle": str(e)} | |
| return { | |
| "fuente": "ChromaDB · NVIDIA NIM embeddings · búsqueda multi-colección", | |
| "modelo": MODEL_NAME, | |
| "query": query, | |
| "fuentes": resultados, | |
| } | |
| # ─── Estado del índice ──────────────────────────────────────────────────────── | |
| def get_index_status() -> dict: | |
| status = {} | |
| for col_name in ["banxico_historico", "banxico_actual", "worldbank", "inegi", "ift"]: | |
| try: | |
| col = _get_collection(col_name) | |
| count = col.count() | |
| except Exception: | |
| count = 0 | |
| idx = _index_status.get(col_name, {}) | |
| status[col_name] = { | |
| "indexado": idx.get("indexed", False), | |
| "documentos": count, | |
| "ultimo_ts": idx.get("ts", "nunca"), | |
| "hash": idx.get("hash", "—"), | |
| } | |
| return status | |
| def init_chroma(force: bool = False) -> dict: | |
| """Punto de entrada único para inicializar ChromaDB al arrancar la app.""" | |
| return index_all(force=force) | |
| if __name__ == "__main__": | |
| print("Inicializando ChromaDB con NVIDIA NIM embeddings...") | |
| result = init_chroma(force=True) | |
| print(json.dumps(result, indent=2, ensure_ascii=False)) | |
| print("\nPrueba búsqueda IFT:") | |
| r = search_chroma("operadores 5G México", "ift", n_results=3) | |
| print(json.dumps(r, indent=2, ensure_ascii=False)) | |
| def reset_collections() -> dict: | |
| """ | |
| Borra físicamente /data/chroma_db/ y limpia cache en memoria. | |
| Necesario cuando cambia el modelo de embeddings (dimensión 384→1024). | |
| """ | |
| global _collections, _chroma_client | |
| import shutil | |
| # 1. Cerrar y destruir cliente ChromaDB en memoria | |
| try: | |
| if _chroma_client is not None: | |
| del _chroma_client | |
| except Exception: | |
| pass | |
| _chroma_client = None | |
| _collections = {} | |
| _index_status.clear() | |
| # 2. Borrar archivos físicos en disco | |
| if os.path.exists(CHROMA_PATH): | |
| try: | |
| shutil.rmtree(CHROMA_PATH) | |
| except Exception as e: | |
| return {"status": "error", "reason": f"No se pudo borrar {CHROMA_PATH}: {e}"} | |
| # 3. Recrear carpeta vacía con permisos correctos | |
| os.makedirs(CHROMA_PATH, exist_ok=True) | |
| os.chmod(CHROMA_PATH, 0o777) | |
| # 4. Crear cliente nuevo desde cero | |
| import chromadb | |
| _chroma_client = chromadb.EphemeralClient() | |
| return { | |
| "status": "ok", | |
| "deleted": CHROMA_PATH, | |
| "mensaje": "ChromaDB reseteado completamente. Cliente recreado. Listo para re-indexar.", | |
| } | |
| def init_chroma_fresh() -> dict: | |
| """ | |
| Borra físicamente ChromaDB y re-indexa desde cero con NVIDIA NIM embeddings. | |
| Usar cuando se cambia el modelo de embeddings (ej: 384→1024 dimensiones). | |
| """ | |
| reset_result = reset_collections() | |
| if reset_result.get("status") == "error": | |
| return {"status": "error", "reset": reset_result} | |
| index_result = index_all(force=True) | |
| return { | |
| "status": "ok" if index_result.get("errores", 1) == 0 else "partial", | |
| "reset": reset_result, | |
| "index": index_result, | |
| } |