Spaces:
Sleeping
Sleeping
Upload 6 files
Browse files- modules/agent_orchestrator.py +841 -0
- modules/agent_tools.py +764 -0
- modules/critic_loop.py +440 -0
- modules/observability.py +330 -0
- modules/rag_chroma.py +828 -0
- modules/semantic_cache.py +317 -0
modules/agent_orchestrator.py
ADDED
|
@@ -0,0 +1,841 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · Orquestador v4 — Multi-Modelo Anti-Alucinación
|
| 3 |
+
=========================================================
|
| 4 |
+
Modelos:
|
| 5 |
+
- LLaMA 3.1 8B → Clasificador + Validador
|
| 6 |
+
- LLaMA 3.1 70B → Análisis numérico, narrativo y síntesis
|
| 7 |
+
- Mixtral 8x7B → Dashboard
|
| 8 |
+
v4.1: Integración read_ift (telecomunicaciones México)
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
import asyncio
|
| 14 |
+
import requests
|
| 15 |
+
from modules.agent_tools import TOOLS, execute_tool
|
| 16 |
+
from modules.critic_loop import critic_loop, load_raw_data_for_validation
|
| 17 |
+
from modules.semantic_cache import get_cached, set_cached
|
| 18 |
+
from modules.observability import QueryTrace
|
| 19 |
+
|
| 20 |
+
NVIDIA_API_KEY = os.getenv("NVIDIA_API_KEY", "")
|
| 21 |
+
NVIDIA_BASE_URL = "https://integrate.api.nvidia.com/v1"
|
| 22 |
+
|
| 23 |
+
# Groq — Llama 3.3 70B numérico — más rápido
|
| 24 |
+
GROQ_API_KEY = os.getenv("GROQ_API_KEY", "")
|
| 25 |
+
GROQ_BASE_URL = "https://api.groq.com/openai/v1"
|
| 26 |
+
|
| 27 |
+
# Tavily — búsqueda web en tiempo real
|
| 28 |
+
TAVILY_API_KEY = os.getenv("TAVILY_API_KEY", "")
|
| 29 |
+
TAVILY_URL = "https://api.tavily.com/search"
|
| 30 |
+
|
| 31 |
+
# ─── Modelos ──────────────────────────────────────────────────────────────────
|
| 32 |
+
MODEL_CLASSIFIER = "meta/llama-3.1-8b-instruct" # Clasificador SIMPLE/NUMÉRICA/NARRATIVA
|
| 33 |
+
MODEL_NUMERICAL = "llama-3.3-70b-versatile" # Análisis numérico via Groq LPU — 5-10x más rápido que NVIDIA NIM
|
| 34 |
+
MODEL_NARRATIVE = "meta/llama-3.1-70b-instruct" # Análisis narrativo y contexto complejo
|
| 35 |
+
MODEL_VALIDATOR = "meta/llama-3.1-8b-instruct" # Validador anti-alucinación
|
| 36 |
+
MODEL_SYNTHESIS = "meta/llama-3.1-70b-instruct" # Síntesis final bilingüe
|
| 37 |
+
|
| 38 |
+
# ─── System prompts ───────────────────────────────────────────────────────────
|
| 39 |
+
SYSTEM_STRICT = """Eres el Agente Económico de la plataforma MX Data Intelligence.
|
| 40 |
+
Tu objetivo es proveer datos macroeconómicos y de telecomunicaciones de México 100% verídicos.
|
| 41 |
+
Fuentes oficiales: Banxico, World Bank, INEGI, IFT.
|
| 42 |
+
|
| 43 |
+
REGLAS ESTRICTAS — sin excepciones:
|
| 44 |
+
1. NUNCA uses tu memoria interna para datos numéricos — solo tool results.
|
| 45 |
+
2. PROHIBIDO repetir el mismo decimal en años consecutivos sin que la fuente lo confirme.
|
| 46 |
+
3. Si el dato no existe o no fue publicado → responde exactamente: "El dato oficial para [período] aún no ha sido publicado por las instituciones correspondientes."
|
| 47 |
+
4. Cada número en tu respuesta DEBE tener su fuente explícita (Banxico/World Bank/INEGI/IFT) y fecha.
|
| 48 |
+
5. Si la herramienta retorna error → dilo explícitamente, no improvises.
|
| 49 |
+
6. Responde en el mismo idioma de la pregunta (español o inglés).
|
| 50 |
+
7. El campo inflacion_ipc de Banxico es el ÍNDICE INPC base 2018=100, NO es el porcentaje de inflación anual. NUNCA lo presentes como porcentaje.
|
| 51 |
+
8. Para inflación anual % usa SOLO el indicador inflacion_pct del World Bank. Si no está disponible → escribe exactamente: "Inflación anual: dato no disponible en la extracción actual."
|
| 52 |
+
9. PIB siempre en billones: $1.856 billones USD (1 billón = 1,000,000 millones).
|
| 53 |
+
10. ESCALA WORLD BANK — CRÍTICO: SIEMPRE usa valor_fmt_es para presentar datos del World Bank. Exportaciones, importaciones y remesas son "miles de millones USD", NO "billones USD".
|
| 54 |
+
11. DATOS 5G MÉXICO — CRÍTICO: El IFT NO publica datos de penetración 5G desagregados en sus reportes de Datos Oportunos. Si preguntan por penetración 5G: indica que el IFT no publica ese dato desagregado, y que la estimación de cobertura poblacional GSMA 2024 es ~3.5% (no usuarios activos). NO cites porcentajes de penetración 5G del IFT porque ese dato no existe en el reporte oficial.
|
| 55 |
+
10. BÚSQUEDA WEB — search_web() disponible via Tavily API. Úsala cuando: (a) el dato no está en el bucket, (b) preguntan por regulatorio reciente (CRT/ATDT), (c) portabilidad más reciente que H1 2025, (d) noticias de operadores. Siempre cita la URL fuente.
|
| 56 |
+
11. Para datos de telecomunicaciones (Telcel, AT&T, Movistar, Telmex, 5G, etc.) usa SIEMPRE read_ift — nunca inventes participaciones de mercado.
|
| 57 |
+
12. OPERADORES 5G MÉXICO — CRÍTICO ANTI-ALUCINACIÓN:
|
| 58 |
+
5G REAL confirmado: Telcel (red propia) y AT&T (red propia). Movistar opera 5G vía acuerdo con AT&T.
|
| 59 |
+
OMVs: Bait/Diri/PilloFon usan Altán Redes = 4.5G LTE, NO 5G. NUNCA digas que Bait, Diri o PilloFon tienen 5G.
|
| 60 |
+
Megacable y Virgin Mobile: depende de zona — NO afirmes que tienen 5G si no está en el dato oficial.
|
| 61 |
+
Si no hay dato oficial confirmado → di exactamente: "El IFT Q1 2025 no certifica el acceso 5G de este OMV."
|
| 62 |
+
13. REGULADOR TELECOMUNICACIONES — CRÍTICO: El IFT sigue siendo el regulador oficial de telecomunicaciones en México. En 2025 quedó reorganizado y adscrito a la ATDT pero NO fue extinto — sigue operando. NUNCA digas que el IFT fue extinto o que la CRT lo reemplazó. Los datos de nuestro bucket son del IFT Q1 2025. SIEMPRE que respondas preguntas de telecomunicaciones agrega: "Fuente: IFT Q1 2025."
|
| 63 |
+
14. PORTABILIDAD — CRÍTICO ANTI-ALUCINACIÓN:
|
| 64 |
+
DATOS REALES disponibles (fuente: consolidado semestral IFT, reportado julio 2025 — NO es Q1 2025):
|
| 65 |
+
- Movistar H1 2025 (ene-jun 2025): +32,667 líneas netas
|
| 66 |
+
- AT&T H1 2025 (ene-jun 2025): -752,664 líneas netas
|
| 67 |
+
- Telcel H1 2025: desaceleración 42% vs año anterior
|
| 68 |
+
- Bait (OMV) H1 2025: +244,024 líneas netas
|
| 69 |
+
CRÍTICO: Estos datos son del CONSOLIDADO SEMESTRAL (H1 = primer semestre ene-jun 2025), NO del reporte Q1 (enero-marzo).
|
| 70 |
+
NUNCA atribuyas datos semestrales a un reporte Q1. Son períodos diferentes.
|
| 71 |
+
Si preguntan por Q1 específico: "El reporte IFT Q1 2025 no desglosa saldos numéricos de portabilidad. El dato disponible es el consolidado H1 2025."
|
| 72 |
+
Si preguntan por período no disponible: "No tengo ese dato. Consulta ift.org.mx/portabilidad"
|
| 73 |
+
15. REGLA GENERAL ANTI-ALUCINACIÓN: Si no tienes el dato exacto en el bucket, responde: "No cuento con ese dato en mi fuente actual. Te recomiendo consultar [fuente oficial]." NUNCA inventes justificaciones de por qué no tienes el dato.
|
| 74 |
+
"""
|
| 75 |
+
|
| 76 |
+
PROMPT_CLASSIFIER = """Clasifica esta pregunta económica en UNA de estas 3 categorías:
|
| 77 |
+
|
| 78 |
+
SIMPLE - pregunta sobre dato ACTUAL o RECIENTE sin fecha específica pasada
|
| 79 |
+
Ejemplos: "¿Cuál es el tipo de cambio hoy?", "¿Cómo está la tasa TIIE?", "¿Cuál es el PIB?"
|
| 80 |
+
|
| 81 |
+
NUMERICA - pregunta sobre histórico, año específico (2024/2025/2026), promedio, estadísticas, tendencia, máximo, mínimo, evolución
|
| 82 |
+
Ejemplos: "¿Cuál fue el promedio en 2025?", "tipo de cambio durante 2024", "estadísticas SF43718", "mínimo histórico"
|
| 83 |
+
|
| 84 |
+
NARRATIVA - análisis complejo, comparaciones entre fuentes, impactos, escenarios, explicaciones
|
| 85 |
+
Ejemplos: "¿Cómo afecta la inflación al PIB?", "compara Banxico con World Bank", "análisis económico completo"
|
| 86 |
+
|
| 87 |
+
REGLA IMPORTANTE: Si la pregunta menciona un año (2024, 2025, 2026) o palabras como promedio/histórico/estadísticas/tendencia/mínimo/máximo → siempre es NUMERICA.
|
| 88 |
+
|
| 89 |
+
Responde SOLO con una palabra: SIMPLE, NUMERICA o NARRATIVA."""
|
| 90 |
+
|
| 91 |
+
PROMPT_VALIDATOR = """Eres un validador silencioso de datos económicos.
|
| 92 |
+
Tu tarea: revisar y corregir la respuesta. Devuelve SOLO la respuesta corregida — sin explicaciones, sin comentarios, sin mencionar qué corregiste.
|
| 93 |
+
|
| 94 |
+
Reglas de corrección:
|
| 95 |
+
1. Agrega fuente explícita (Banxico/World Bank/INEGI/IFT) a cada número que no la tenga
|
| 96 |
+
2. Si un decimal parece inventado, márcalo como [dato no verificado]
|
| 97 |
+
3. No agregues datos nuevos — solo corrige formato y fuentes
|
| 98 |
+
4. Devuelve la respuesta lista para mostrar al usuario final"""
|
| 99 |
+
|
| 100 |
+
MULTI_KEYWORDS = [
|
| 101 |
+
"cruzando", "cruza", "todas las fuentes", "salud económica",
|
| 102 |
+
"resumen económico", "análisis completo", "los tres", "las tres",
|
| 103 |
+
"multi", "completo", "integral", "compara", "vs", "versus"
|
| 104 |
+
]
|
| 105 |
+
|
| 106 |
+
# ── Keywords telecomunicaciones ───────────────────────────────────────────────
|
| 107 |
+
TELCO_KEYWORDS = [
|
| 108 |
+
"telcel", "att", "at&t", "movistar", "telmex", "telnorte",
|
| 109 |
+
"telecomunicaciones", "telco", "telefonia movil", "telefonía móvil",
|
| 110 |
+
"5g", "4g", "lte", "cobertura celular", "cobertura movil",
|
| 111 |
+
"portabilidad numerica", "portabilidad numérica",
|
| 112 |
+
"banda ancha", "internet movil", "internet móvil",
|
| 113 |
+
"operador movil", "operador móvil", "lineas moviles", "líneas móviles",
|
| 114 |
+
"ift", "instituto federal de telecomunicaciones",
|
| 115 |
+
"participacion mercado movil", "participación mercado móvil",
|
| 116 |
+
"tv restringida", "television restringida", "televisión restringida",
|
| 117 |
+
"megacable", "izzi", "sky", "total play", "totalplay",
|
| 118 |
+
"agente economico preponderante", "aep telecom",
|
| 119 |
+
]
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
| 123 |
+
|
| 124 |
+
def needs_multi_source(question: str) -> bool:
|
| 125 |
+
q = question.lower()
|
| 126 |
+
has_banxico = any(k in q for k in ["banxico", "tipo de cambio", "tasa", "tiie", "inflacion", "inflación"])
|
| 127 |
+
has_wb = any(k in q for k in ["world bank", "pib", "exportaciones", "deuda", "worldbank"])
|
| 128 |
+
has_inegi = any(k in q for k in ["inegi", "poblacion", "población", "ocupacion", "igae"])
|
| 129 |
+
has_multi = any(k in q for k in MULTI_KEYWORDS)
|
| 130 |
+
return sum([has_banxico, has_wb, has_inegi]) >= 2 or has_multi
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
# Global trace reference for token tracking
|
| 134 |
+
_current_trace = None
|
| 135 |
+
|
| 136 |
+
def call_model(model: str, messages: list, max_tokens: int = 700) -> tuple:
|
| 137 |
+
"""Llamada genérica a cualquier modelo vía NVIDIA API."""
|
| 138 |
+
headers = {
|
| 139 |
+
"Authorization": f"Bearer {NVIDIA_API_KEY}",
|
| 140 |
+
"Content-Type": "application/json",
|
| 141 |
+
}
|
| 142 |
+
payload = {
|
| 143 |
+
"model": model,
|
| 144 |
+
"messages": messages,
|
| 145 |
+
"temperature": 0.1,
|
| 146 |
+
"max_tokens": max_tokens,
|
| 147 |
+
"stream": False,
|
| 148 |
+
}
|
| 149 |
+
for attempt in range(3):
|
| 150 |
+
try:
|
| 151 |
+
r = requests.post(
|
| 152 |
+
f"{NVIDIA_BASE_URL}/chat/completions",
|
| 153 |
+
headers=headers,
|
| 154 |
+
json=payload,
|
| 155 |
+
timeout=180,
|
| 156 |
+
)
|
| 157 |
+
r.raise_for_status()
|
| 158 |
+
data = r.json()
|
| 159 |
+
result = data["choices"][0]["message"].get("content", "")
|
| 160 |
+
# Track tokens from API response or estimate
|
| 161 |
+
if _current_trace:
|
| 162 |
+
usage = data.get("usage", {})
|
| 163 |
+
input_tokens = usage.get("prompt_tokens", sum(len(m.get("content","")) for m in messages) // 4)
|
| 164 |
+
output_tokens = usage.get("completion_tokens", len(result) // 4)
|
| 165 |
+
_current_trace.record_tokens(model, input_tokens, output_tokens)
|
| 166 |
+
return result, None
|
| 167 |
+
except requests.exceptions.Timeout:
|
| 168 |
+
if attempt < 2:
|
| 169 |
+
continue
|
| 170 |
+
return None, "timeout"
|
| 171 |
+
except Exception as e:
|
| 172 |
+
return None, str(e)
|
| 173 |
+
return None, "timeout"
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def call_model_groq(model: str, messages: list, max_tokens: int = 700) -> tuple:
|
| 177 |
+
"""Llamada a Groq API para Qwen 2.5 72B — más rápido para análisis numérico."""
|
| 178 |
+
if not GROQ_API_KEY:
|
| 179 |
+
# Fallback a NVIDIA si no hay Groq key
|
| 180 |
+
return call_model("meta/llama-3.1-70b-instruct", messages, max_tokens)
|
| 181 |
+
headers = {
|
| 182 |
+
"Authorization": f"Bearer {GROQ_API_KEY}",
|
| 183 |
+
"Content-Type": "application/json",
|
| 184 |
+
}
|
| 185 |
+
payload = {
|
| 186 |
+
"model": "llama-3.3-70b-versatile",
|
| 187 |
+
"messages": messages,
|
| 188 |
+
"temperature": 0.1,
|
| 189 |
+
"max_tokens": max_tokens,
|
| 190 |
+
"stream": False,
|
| 191 |
+
}
|
| 192 |
+
for attempt in range(3):
|
| 193 |
+
try:
|
| 194 |
+
r = requests.post(
|
| 195 |
+
f"{GROQ_BASE_URL}/chat/completions",
|
| 196 |
+
headers=headers,
|
| 197 |
+
json=payload,
|
| 198 |
+
timeout=30,
|
| 199 |
+
)
|
| 200 |
+
r.raise_for_status()
|
| 201 |
+
data = r.json()
|
| 202 |
+
result = data["choices"][0]["message"].get("content", "")
|
| 203 |
+
if _current_trace:
|
| 204 |
+
usage = data.get("usage", {})
|
| 205 |
+
input_tokens = usage.get("prompt_tokens", sum(len(m.get("content","")) for m in messages) // 4)
|
| 206 |
+
output_tokens = usage.get("completion_tokens", len(result) // 4)
|
| 207 |
+
_current_trace.record_tokens("groq/llama-3.3-70b-versatile", input_tokens, output_tokens)
|
| 208 |
+
return result, None
|
| 209 |
+
except requests.exceptions.Timeout:
|
| 210 |
+
if attempt < 2:
|
| 211 |
+
continue
|
| 212 |
+
# Fallback a NVIDIA en timeout
|
| 213 |
+
return call_model("meta/llama-3.1-70b-instruct", messages, max_tokens)
|
| 214 |
+
except Exception as e:
|
| 215 |
+
if attempt < 2:
|
| 216 |
+
continue
|
| 217 |
+
# Fallback a NVIDIA en error
|
| 218 |
+
return call_model("meta/llama-3.1-70b-instruct", messages, max_tokens)
|
| 219 |
+
return call_model("meta/llama-3.1-70b-instruct", messages, max_tokens)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def search_web(query: str, max_results: int = 3) -> dict:
|
| 223 |
+
"""Búsqueda web en tiempo real via Tavily API."""
|
| 224 |
+
if not TAVILY_API_KEY:
|
| 225 |
+
return {"error": "TAVILY_API_KEY no configurada", "results": []}
|
| 226 |
+
try:
|
| 227 |
+
payload = {
|
| 228 |
+
"api_key": TAVILY_API_KEY,
|
| 229 |
+
"query": query,
|
| 230 |
+
"max_results": max_results,
|
| 231 |
+
"search_depth": "basic",
|
| 232 |
+
"include_answer": True,
|
| 233 |
+
"include_domains": [
|
| 234 |
+
"ift.org.mx", "crt.gob.mx", "atdt.gob.mx",
|
| 235 |
+
"banxico.org.mx", "inegi.org.mx",
|
| 236 |
+
"eleconomista.com.mx", "expansion.mx",
|
| 237 |
+
"milenio.com", "elfinanciero.com.mx"
|
| 238 |
+
]
|
| 239 |
+
}
|
| 240 |
+
r = requests.post(TAVILY_URL, json=payload, timeout=15)
|
| 241 |
+
r.raise_for_status()
|
| 242 |
+
data = r.json()
|
| 243 |
+
results = []
|
| 244 |
+
for item in data.get("results", [])[:max_results]:
|
| 245 |
+
results.append({
|
| 246 |
+
"titulo": item.get("title", ""),
|
| 247 |
+
"url": item.get("url", ""),
|
| 248 |
+
"snippet": item.get("content", "")[:400],
|
| 249 |
+
"fecha": item.get("published_date", "")
|
| 250 |
+
})
|
| 251 |
+
return {
|
| 252 |
+
"answer": data.get("answer", ""),
|
| 253 |
+
"results": results,
|
| 254 |
+
"query": query
|
| 255 |
+
}
|
| 256 |
+
except Exception as e:
|
| 257 |
+
return {"error": str(e), "results": []}
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def classify_question(question: str) -> str:
|
| 261 |
+
"""LLaMA 8B clasifica la pregunta: SIMPLE / NUMERICA / NARRATIVA."""
|
| 262 |
+
msgs = [
|
| 263 |
+
{"role": "system", "content": PROMPT_CLASSIFIER},
|
| 264 |
+
{"role": "user", "content": question}
|
| 265 |
+
]
|
| 266 |
+
result, error = call_model(MODEL_CLASSIFIER, msgs, max_tokens=10)
|
| 267 |
+
if error or not result:
|
| 268 |
+
return "NARRATIVA"
|
| 269 |
+
result = result.strip().upper()
|
| 270 |
+
if "SIMPLE" in result:
|
| 271 |
+
return "SIMPLE"
|
| 272 |
+
elif "NUMERICA" in result or "NUMÉRICA" in result:
|
| 273 |
+
return "NUMERICA"
|
| 274 |
+
else:
|
| 275 |
+
return "NARRATIVA"
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
async def fetch_tool_async(tool_name: str, args: dict) -> tuple:
|
| 279 |
+
"""Ejecuta una tool de forma asíncrona."""
|
| 280 |
+
loop = asyncio.get_event_loop()
|
| 281 |
+
result = await loop.run_in_executor(None, execute_tool, tool_name, args)
|
| 282 |
+
return tool_name, result
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
async def prefetch_all_parallel(placeholder=None) -> str:
|
| 286 |
+
"""Ejecuta las 3 tools en paralelo con asyncio.gather()."""
|
| 287 |
+
if placeholder:
|
| 288 |
+
placeholder.markdown("*Consultando Banxico + World Bank + INEGI en paralelo...*")
|
| 289 |
+
|
| 290 |
+
tasks = [
|
| 291 |
+
fetch_tool_async("read_banxico", {"indicador": "todos"}),
|
| 292 |
+
fetch_tool_async("read_worldbank", {"categoria": "economia"}),
|
| 293 |
+
fetch_tool_async("read_inegi", {"indicador": "todos"}),
|
| 294 |
+
]
|
| 295 |
+
|
| 296 |
+
results = await asyncio.gather(*tasks, return_exceptions=True)
|
| 297 |
+
|
| 298 |
+
parts = []
|
| 299 |
+
labels = {"read_banxico": "BANXICO", "read_worldbank": "WORLD BANK", "read_inegi": "INEGI"}
|
| 300 |
+
|
| 301 |
+
BILLONES_KEYS = ["pib_usd", "exportaciones_usd", "importaciones_usd", "remesas_usd", "cuenta_corriente_usd"]
|
| 302 |
+
PCT_KEYS = ["inflacion_pct", "desempleo_pct", "deuda_pct_pib", "crecimiento_pib_pct", "pobreza_pct",
|
| 303 |
+
"alfabetizacion_pct", "acceso_electricidad_pct", "area_forestal_pct"]
|
| 304 |
+
|
| 305 |
+
def fmt_valor(k, v, meta=None):
|
| 306 |
+
# FIX v4: usar valor_fmt_es del World Bank si está disponible
|
| 307 |
+
if isinstance(meta, dict) and "valor_fmt_es" in meta:
|
| 308 |
+
return meta["valor_fmt_es"]
|
| 309 |
+
try:
|
| 310 |
+
n = float(str(v).replace(",", ""))
|
| 311 |
+
if k in BILLONES_KEYS:
|
| 312 |
+
if abs(n) > 1_000_000_000_000:
|
| 313 |
+
return f"{n/1_000_000_000_000:,.3f} billones USD"
|
| 314 |
+
elif abs(n) > 1_000_000_000:
|
| 315 |
+
return f"{n/1_000_000_000:,.1f} miles de millones USD"
|
| 316 |
+
elif k in PCT_KEYS:
|
| 317 |
+
return f"{n:.2f}%"
|
| 318 |
+
elif k in ["inflacion_ipc", "SP1"]:
|
| 319 |
+
return f"{n:.3f} (índice INPC base 2018=100 — NO es % de inflación)"
|
| 320 |
+
elif k in ["tipo_cambio_usd_mxn", "SF43718"]:
|
| 321 |
+
return f"{n:.4f} MXN/USD"
|
| 322 |
+
elif k in ["tiie_28_dias", "tasa_objetivo", "SF43783", "SF61745"]:
|
| 323 |
+
return f"{n:.4f}%"
|
| 324 |
+
elif n > 1_000_000:
|
| 325 |
+
return f"{n:,.0f}"
|
| 326 |
+
return v
|
| 327 |
+
except (TypeError, ValueError):
|
| 328 |
+
return v
|
| 329 |
+
|
| 330 |
+
for tool_name, result in results:
|
| 331 |
+
label = labels.get(tool_name, tool_name)
|
| 332 |
+
if isinstance(result, Exception):
|
| 333 |
+
parts.append(f"[{label}]: Error - {result}")
|
| 334 |
+
continue
|
| 335 |
+
try:
|
| 336 |
+
data = json.loads(result)
|
| 337 |
+
datos = data.get("datos", data.get("datos_disponibles", data))
|
| 338 |
+
if isinstance(datos, dict):
|
| 339 |
+
resumen = {}
|
| 340 |
+
for k, v in datos.items():
|
| 341 |
+
if k in ["fuente", "timestamp", "archivo", "pais", "nota_inflacion"]:
|
| 342 |
+
continue
|
| 343 |
+
if isinstance(v, dict):
|
| 344 |
+
val = v.get("valor_formateado", v.get("valor", str(v)))
|
| 345 |
+
resumen[k] = fmt_valor(k, val)
|
| 346 |
+
elif not isinstance(v, (list, dict)):
|
| 347 |
+
resumen[k] = fmt_valor(k, v)
|
| 348 |
+
parts.append(f"[{label}]: " + ", ".join(f"{k}={v}" for k, v in list(resumen.items())[:10]))
|
| 349 |
+
else:
|
| 350 |
+
parts.append(f"[{label}]: {str(datos)[:300]}")
|
| 351 |
+
except Exception as e:
|
| 352 |
+
parts.append(f"[{label}]: Error procesando - {e}")
|
| 353 |
+
|
| 354 |
+
return "\n".join(parts)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def validate_response(response: str, data_context: str, question: str = "") -> str:
|
| 358 |
+
"""
|
| 359 |
+
Validación en 2 pasos:
|
| 360 |
+
1. Critic-Loop Python — verificación matemática determinística (~1ms)
|
| 361 |
+
2. LLaMA 8B — validación semántica y de fuentes
|
| 362 |
+
"""
|
| 363 |
+
# Paso 1: Critic-Loop matemático (Python, sin LLM)
|
| 364 |
+
raw_data = load_raw_data_for_validation()
|
| 365 |
+
final, iterations, n_errors = critic_loop(
|
| 366 |
+
draft=response,
|
| 367 |
+
question=question,
|
| 368 |
+
raw_data=raw_data,
|
| 369 |
+
call_model_fn=call_model,
|
| 370 |
+
model=MODEL_VALIDATOR,
|
| 371 |
+
system_prompt=SYSTEM_STRICT,
|
| 372 |
+
)
|
| 373 |
+
# Si el critic loop corrigió algo, usar esa versión
|
| 374 |
+
response = final
|
| 375 |
+
|
| 376 |
+
# Paso 2: LLaMA 8B — validación semántica
|
| 377 |
+
msgs = [
|
| 378 |
+
{"role": "system", "content": PROMPT_VALIDATOR},
|
| 379 |
+
{"role": "user", "content": f"DATOS REALES:\n{data_context}\n\nRESPUESTA A VALIDAR:\n{response}"}
|
| 380 |
+
]
|
| 381 |
+
validated, error = call_model(MODEL_VALIDATOR, msgs, max_tokens=800)
|
| 382 |
+
if error or not validated:
|
| 383 |
+
return response
|
| 384 |
+
return validated
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
def synthesize_response(data_context: str, question: str, placeholder=None) -> str:
|
| 388 |
+
"""LLaMA 70B redacta la respuesta final bilingüe."""
|
| 389 |
+
if placeholder:
|
| 390 |
+
placeholder.markdown("*Redactando respuesta final...*")
|
| 391 |
+
|
| 392 |
+
lang_hint = "en inglés" if any(
|
| 393 |
+
w in question.lower() for w in ["what","how","when","where","why","which","is","are","the"]
|
| 394 |
+
) else "en español"
|
| 395 |
+
|
| 396 |
+
msgs = [
|
| 397 |
+
{"role": "system", "content": SYSTEM_STRICT},
|
| 398 |
+
{"role": "user", "content":
|
| 399 |
+
f"Con estos datos reales, responde la pregunta {lang_hint}. "
|
| 400 |
+
f"Menciona la fuente y fecha de cada dato. "
|
| 401 |
+
f"Formato profesional y claro.\n\n"
|
| 402 |
+
f"DATOS:\n{data_context}\n\n"
|
| 403 |
+
f"PREGUNTA: {question}"
|
| 404 |
+
}
|
| 405 |
+
]
|
| 406 |
+
answer, error = call_model(MODEL_SYNTHESIS, msgs, max_tokens=700)
|
| 407 |
+
if error or not answer:
|
| 408 |
+
return f"⚠️ Error en síntesis: {error}"
|
| 409 |
+
return answer
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
# ─── Orquestador principal ────────────────────────────────────────────────────
|
| 413 |
+
|
| 414 |
+
def call_llm_with_tools(messages: list, stream_placeholder=None) -> str:
|
| 415 |
+
if not NVIDIA_API_KEY:
|
| 416 |
+
return "⚠️ Configura NVIDIA_API_KEY en Settings → Secrets."
|
| 417 |
+
|
| 418 |
+
user_question = messages[-1].get("content", "") if messages else ""
|
| 419 |
+
|
| 420 |
+
# ── CACHÉ SEMÁNTICO — antes de cualquier procesamiento ───────────────────
|
| 421 |
+
cached = get_cached(user_question)
|
| 422 |
+
if cached:
|
| 423 |
+
if stream_placeholder:
|
| 424 |
+
stream_placeholder.markdown(cached)
|
| 425 |
+
return cached
|
| 426 |
+
|
| 427 |
+
# ── TRAZA DE OBSERVABILIDAD ──────────────────────────────────────────────
|
| 428 |
+
global _current_trace
|
| 429 |
+
trace = QueryTrace(user_question)
|
| 430 |
+
_current_trace = trace
|
| 431 |
+
|
| 432 |
+
# ── PASO 0: Pre-clasificación Python (sin LLM) ────────────────────────────
|
| 433 |
+
import re as _re_orch
|
| 434 |
+
q_lower = user_question.lower()
|
| 435 |
+
|
| 436 |
+
# ── INEGI: detectar keywords → responder directo con read_inegi ─────────────
|
| 437 |
+
INEGI_KEYWORDS = [
|
| 438 |
+
"poblacion", "población", "personas hay", "cuantas personas",
|
| 439 |
+
"cuántas personas", "habitantes", "igae", "ocupacion", "ocupados",
|
| 440 |
+
"inegi", "inpc", "desocupacion", "desocupación", "actividad economica",
|
| 441 |
+
"actividad económica",
|
| 442 |
+
]
|
| 443 |
+
is_inegi = any(k in q_lower for k in INEGI_KEYWORDS)
|
| 444 |
+
if is_inegi:
|
| 445 |
+
if stream_placeholder:
|
| 446 |
+
stream_placeholder.markdown("*Consulta INEGI → bucket directo...*")
|
| 447 |
+
raw_inegi = execute_tool("read_inegi", {"indicador": "todos"})
|
| 448 |
+
raw_wb = execute_tool("read_worldbank", {"categoria": "social"})
|
| 449 |
+
raw = (
|
| 450 |
+
raw_inegi +
|
| 451 |
+
"\n[WORLD BANK SOCIAL — usa estos datos si INEGI no tiene poblacion_total]:\n" +
|
| 452 |
+
raw_wb +
|
| 453 |
+
"\nINSTRUCCION: Si inegi.json no tiene poblacion_total, usa el dato de World Bank SP.POP.TOTL. "
|
| 454 |
+
"NO digas 'no disponible' si World Bank tiene el dato."
|
| 455 |
+
)
|
| 456 |
+
# Sin validador — respuesta directa del sintetizador
|
| 457 |
+
msgs_inegi = [
|
| 458 |
+
{"role": "system", "content": SYSTEM_STRICT},
|
| 459 |
+
{"role": "user", "content":
|
| 460 |
+
f"DATOS REALES:\n{raw[:2500]}\n\n"
|
| 461 |
+
f"INSTRUCCION CRÍTICA: En los datos anteriores busca 'poblacion_total'. "
|
| 462 |
+
f"Si encuentras '128,455,567' o cualquier número de población, ÚSALO. "
|
| 463 |
+
f"NO digas 'no disponible'. Responde en español.\n\n"
|
| 464 |
+
f"PREGUNTA: {user_question}"
|
| 465 |
+
}
|
| 466 |
+
]
|
| 467 |
+
answer_inegi, _ = call_model(MODEL_SYNTHESIS, msgs_inegi, max_tokens=400)
|
| 468 |
+
result_inegi = answer_inegi or raw[:500]
|
| 469 |
+
if stream_placeholder:
|
| 470 |
+
stream_placeholder.markdown(result_inegi)
|
| 471 |
+
trace.set_route("INEGI")
|
| 472 |
+
trace.finish()
|
| 473 |
+
_current_trace = None
|
| 474 |
+
set_cached(user_question, result_inegi)
|
| 475 |
+
return result_inegi
|
| 476 |
+
|
| 477 |
+
# ── WORLD BANK: detectar keywords → responder directo con read_worldbank ────
|
| 478 |
+
WB_KEYWORDS = [
|
| 479 |
+
"exportaciones", "exportacion", "importaciones", "importacion",
|
| 480 |
+
"world bank", "banco mundial", "worldbank", "remesas",
|
| 481 |
+
"deuda publica", "deuda pública", "pobreza", "desigualdad",
|
| 482 |
+
"co2", "emisiones", "area forestal", "electricidad acceso",
|
| 483 |
+
"control corrupcion", "efectividad gobierno",
|
| 484 |
+
"sp.pop.totl", "indicadores world bank", "indicadores banco mundial",
|
| 485 |
+
"resumen world bank", "resumen banco mundial",
|
| 486 |
+
]
|
| 487 |
+
is_worldbank = any(k in q_lower for k in WB_KEYWORDS)
|
| 488 |
+
if is_worldbank:
|
| 489 |
+
if stream_placeholder:
|
| 490 |
+
stream_placeholder.markdown("*Consulta World Bank → bucket directo...*")
|
| 491 |
+
# Detectar categoría específica
|
| 492 |
+
if any(k in q_lower for k in ["exportacion", "importacion", "remesa", "cuenta corriente", "comercio"]):
|
| 493 |
+
cat_wb = "comercio"
|
| 494 |
+
elif any(k in q_lower for k in ["pobreza", "poblacion", "desempleo", "alfabetizacion", "social"]):
|
| 495 |
+
cat_wb = "social"
|
| 496 |
+
elif any(k in q_lower for k in ["co2", "emision", "forestal", "energia", "electricidad"]):
|
| 497 |
+
cat_wb = "medio_ambiente"
|
| 498 |
+
elif any(k in q_lower for k in ["corrupcion", "gobierno", "gobernanza"]):
|
| 499 |
+
cat_wb = "gobernanza"
|
| 500 |
+
elif any(k in q_lower for k in ["energia", "petroleo"]):
|
| 501 |
+
cat_wb = "energia"
|
| 502 |
+
else:
|
| 503 |
+
cat_wb = "todos" # default: todos los indicadores
|
| 504 |
+
raw = execute_tool("read_worldbank", {"categoria": cat_wb})
|
| 505 |
+
# Si pregunta sobre población, agregar categoría social explícitamente
|
| 506 |
+
if any(k in q_lower for k in ["poblacion", "población", "personas", "habitantes"]):
|
| 507 |
+
raw_social = execute_tool("read_worldbank", {"categoria": "social"})
|
| 508 |
+
raw = raw + "\n[SOCIAL WB]:\n" + raw_social
|
| 509 |
+
raw += (
|
| 510 |
+
"\nNOTA: En los datos de World Bank, busca el indicador 'poblacion_total' "
|
| 511 |
+
"o 'SP.POP.TOTL'. La población total de México es aproximadamente 130 millones "
|
| 512 |
+
"de personas según el indicador SP.POP.TOTL del Banco Mundial. "
|
| 513 |
+
"USA ese dato para responder."
|
| 514 |
+
)
|
| 515 |
+
trace.set_route("WORLDBANK")
|
| 516 |
+
trace.add_tool("read_worldbank")
|
| 517 |
+
result_wb = synthesize_response(raw[:3000], user_question, stream_placeholder)
|
| 518 |
+
trace.finish()
|
| 519 |
+
set_cached(user_question, result_wb)
|
| 520 |
+
return result_wb
|
| 521 |
+
|
| 522 |
+
# ── TELCO: detectar keywords IFT → responder directo con read_ift ─────────
|
| 523 |
+
is_telco = any(k in q_lower for k in TELCO_KEYWORDS)
|
| 524 |
+
if is_telco:
|
| 525 |
+
if stream_placeholder:
|
| 526 |
+
stream_placeholder.markdown("*Consulta telco → IFT datos abiertos...*")
|
| 527 |
+
# Preguntas de análisis cruzado → siempre "todo" para tener todos los datos
|
| 528 |
+
is_cross = any(k in q_lower for k in [
|
| 529 |
+
"relacion", "relación", "entre", "compara", "vs", "versus",
|
| 530 |
+
"y el", "e ingreso", "e ingresos", "analisis", "análisis", "resumen"
|
| 531 |
+
])
|
| 532 |
+
if is_cross:
|
| 533 |
+
categoria_ift = "todo"
|
| 534 |
+
elif any(k in q_lower for k in ["5g", "4g", "lte", "cobertura"]):
|
| 535 |
+
categoria_ift = "cobertura_5g"
|
| 536 |
+
elif any(k in q_lower for k in ["participacion", "participación", "mercado", "lineas", "líneas"]):
|
| 537 |
+
categoria_ift = "participacion_mercado_movil"
|
| 538 |
+
elif any(k in q_lower for k in ["ingres", "revenue", "factur"]):
|
| 539 |
+
categoria_ift = "ingresos"
|
| 540 |
+
elif any(k in q_lower for k in ["portabilidad"]):
|
| 541 |
+
# PORTABILIDAD: datos hardcodeados verificados — sin LLM para evitar alucinaciones
|
| 542 |
+
result_port = (
|
| 543 |
+
"**Portabilidad Numérica México — Datos verificados H1 2025**\n\n"
|
| 544 |
+
"| Operador | Saldo H1 2025 | Resultado |\n"
|
| 545 |
+
"|---|---|---|\n"
|
| 546 |
+
"| Movistar | **+32,667 líneas** | Positivo — mejor operador red propia |\n"
|
| 547 |
+
"| Bait (OMV) | **+244,024 líneas** | Positivo — líder captación total |\n"
|
| 548 |
+
"| Telcel | Desaceleración 42% vs 2024 | Disputado |\n"
|
| 549 |
+
"| AT&T | **-752,664 líneas** | Negativo severo |\n\n"
|
| 550 |
+
"**Fuente:** Consolidado semestral IFT, reportado julio 2025 "
|
| 551 |
+
"(H1 = enero-junio 2025 — distinto al reporte Q1 enero-marzo)\n\n"
|
| 552 |
+
"Para datos por trimestre o más recientes consulta: "
|
| 553 |
+
"https://www.ift.org.mx/usuarios-y-audiencias/portabilidad-numerica"
|
| 554 |
+
)
|
| 555 |
+
if stream_placeholder:
|
| 556 |
+
stream_placeholder.markdown(result_port)
|
| 557 |
+
trace.set_route("TELCO_PORTABILIDAD")
|
| 558 |
+
trace.finish()
|
| 559 |
+
_current_trace = None
|
| 560 |
+
set_cached(user_question, result_port)
|
| 561 |
+
return result_port
|
| 562 |
+
elif any(k in q_lower for k in ["tv", "television", "televisión", "restringida"]):
|
| 563 |
+
categoria_ift = "television"
|
| 564 |
+
elif any(k in q_lower for k in ["regulacion", "regulación", "aep", "preponderante"]):
|
| 565 |
+
categoria_ift = "regulacion"
|
| 566 |
+
elif any(k in q_lower for k in ["banda ancha", "internet fijo"]):
|
| 567 |
+
categoria_ift = "internet_fijo"
|
| 568 |
+
else:
|
| 569 |
+
categoria_ift = "todo"
|
| 570 |
+
raw = execute_tool("read_ift", {"categoria": categoria_ift})
|
| 571 |
+
trace.set_route("TELCO")
|
| 572 |
+
trace.add_tool("read_ift")
|
| 573 |
+
result_telco = synthesize_response(raw[:2000], user_question, stream_placeholder)
|
| 574 |
+
trace.finish()
|
| 575 |
+
set_cached(user_question, result_telco)
|
| 576 |
+
return result_telco
|
| 577 |
+
|
| 578 |
+
# ── HISTÓRICO: detectar año o keywords estadísticos → forzar NUMERICA ─────
|
| 579 |
+
years_found = _re_orch.findall(r"\b(202[0-9])\b", user_question)
|
| 580 |
+
is_historical = bool(years_found) or any(k in q_lower for k in [
|
| 581 |
+
"promedio", "histórico", "historico", "average", "mínimo", "minimo",
|
| 582 |
+
"máximo", "maximo", "tendencia", "trend", "durante", "evolución",
|
| 583 |
+
"evolucion", "estadísticas", "estadisticas", "statistics",
|
| 584 |
+
"sf43718", "sf43783", "sf61745", "sp1", "serie", "2024", "2025"
|
| 585 |
+
])
|
| 586 |
+
|
| 587 |
+
# ── PASO 1: Clasificador LLaMA 8B ─────────────────────────────────────────
|
| 588 |
+
if stream_placeholder:
|
| 589 |
+
stream_placeholder.markdown("*Clasificando pregunta...*")
|
| 590 |
+
|
| 591 |
+
if is_historical:
|
| 592 |
+
complexity = "NUMERICA"
|
| 593 |
+
else:
|
| 594 |
+
complexity = classify_question(user_question)
|
| 595 |
+
|
| 596 |
+
# ── PASO 2A: SIMPLE → bucket directo ──────────────────────────────────────
|
| 597 |
+
if complexity == "SIMPLE":
|
| 598 |
+
if stream_placeholder:
|
| 599 |
+
stream_placeholder.markdown("*Consulta simple → bucket directo...*")
|
| 600 |
+
trace.set_route("SIMPLE")
|
| 601 |
+
q = user_question.lower()
|
| 602 |
+
if any(k in q for k in ["poblacion", "población", "personas hay", "habitantes",
|
| 603 |
+
"igae", "ocupacion", "ocupados", "inegi", "inpc",
|
| 604 |
+
"desocupacion", "desocupación"]):
|
| 605 |
+
raw_inegi = execute_tool("read_inegi", {"indicador": "todos"})
|
| 606 |
+
# Si INEGI no tiene poblacion_total → complementar con World Bank
|
| 607 |
+
raw_wb = execute_tool("read_worldbank", {"categoria": "social"})
|
| 608 |
+
raw = raw_inegi + "\n[WORLD BANK COMPLEMENTO]:\n" + raw_wb
|
| 609 |
+
elif any(k in q for k in ["tipo de cambio", "exchange", "usd", "mxn", "tasa", "tiie", "inflacion", "inflación"]):
|
| 610 |
+
raw = execute_tool("read_banxico", {"indicador": "todos"})
|
| 611 |
+
elif any(k in q for k in ["pib", "gdp", "exportaciones", "exportacion", "importaciones",
|
| 612 |
+
"importacion", "deuda", "desempleo", "remesas", "world bank",
|
| 613 |
+
"banco mundial", "crecimiento", "pobreza"]):
|
| 614 |
+
raw = execute_tool("read_worldbank", {"categoria": "todos"})
|
| 615 |
+
else:
|
| 616 |
+
raw = execute_tool("read_banxico", {"indicador": "todos"})
|
| 617 |
+
context = raw[:1500]
|
| 618 |
+
result = synthesize_response(context, user_question, stream_placeholder)
|
| 619 |
+
trace.add_tool("read_banxico" if "banxico" in q else "read_worldbank" if "pib" in q else "read_inegi")
|
| 620 |
+
trace.finish()
|
| 621 |
+
_current_trace = None
|
| 622 |
+
set_cached(user_question, result)
|
| 623 |
+
return result
|
| 624 |
+
|
| 625 |
+
# ── PASO 2B: NUMERICA → LLaMA 70B + histórico ────────────────────────────
|
| 626 |
+
if complexity == "NUMERICA":
|
| 627 |
+
if stream_placeholder:
|
| 628 |
+
stream_placeholder.markdown("*Pregunta numérica → histórico...*")
|
| 629 |
+
trace.set_route("NUMERICA")
|
| 630 |
+
trace.start_stage("historico")
|
| 631 |
+
|
| 632 |
+
q = user_question.lower()
|
| 633 |
+
if any(k in q for k in ["tiie", "interbancaria", "sf43783"]):
|
| 634 |
+
serie_hist = "SF43783"
|
| 635 |
+
elif any(k in q for k in ["tasa objetivo", "objetivo", "sf61745"]):
|
| 636 |
+
serie_hist = "SF61745"
|
| 637 |
+
elif any(k in q for k in ["inflacion", "inflación", "ipc", "sp1"]):
|
| 638 |
+
serie_hist = "SP1"
|
| 639 |
+
else:
|
| 640 |
+
serie_hist = "SF43718"
|
| 641 |
+
|
| 642 |
+
if stream_placeholder:
|
| 643 |
+
stream_placeholder.markdown("*🔍 Búsqueda semántica ChromaDB · NVIDIA NIM H100...*")
|
| 644 |
+
historico_raw = execute_tool("search_historico", {"serie": serie_hist, "query": user_question})
|
| 645 |
+
|
| 646 |
+
try:
|
| 647 |
+
import json as _json
|
| 648 |
+
import re as _re2
|
| 649 |
+
hist_data = _json.loads(historico_raw)
|
| 650 |
+
registros = hist_data.get("resultados", [])
|
| 651 |
+
valores = []
|
| 652 |
+
for r in registros:
|
| 653 |
+
try:
|
| 654 |
+
v = str(r.get("valor", "")).split(" ")[0].replace(",", "")
|
| 655 |
+
valores.append(float(v))
|
| 656 |
+
except Exception:
|
| 657 |
+
pass
|
| 658 |
+
|
| 659 |
+
if valores:
|
| 660 |
+
stats_calc = {
|
| 661 |
+
"promedio": round(sum(valores)/len(valores), 4),
|
| 662 |
+
"minimo": round(min(valores), 4),
|
| 663 |
+
"maximo": round(max(valores), 4),
|
| 664 |
+
"primero": valores[0],
|
| 665 |
+
"ultimo": valores[-1],
|
| 666 |
+
"total": len(valores),
|
| 667 |
+
}
|
| 668 |
+
else:
|
| 669 |
+
stats_calc = hist_data.get("estadisticas", {})
|
| 670 |
+
|
| 671 |
+
años = hist_data.get("años", _re2.findall(r"\b(202[0-9])\b", user_question))
|
| 672 |
+
historico = (
|
| 673 |
+
f"Serie: {hist_data.get('nombre', serie_hist)}\n"
|
| 674 |
+
f"Período consultado: {años}\n"
|
| 675 |
+
f"Total registros: {stats_calc.get('total', len(registros))}\n"
|
| 676 |
+
f"Promedio: {stats_calc.get('promedio')}\n"
|
| 677 |
+
f"Mínimo: {stats_calc.get('minimo')}\n"
|
| 678 |
+
f"Máximo: {stats_calc.get('maximo')}\n"
|
| 679 |
+
f"Primer valor: {stats_calc.get('primero')}\n"
|
| 680 |
+
f"Último valor: {stats_calc.get('ultimo')}\n"
|
| 681 |
+
f"Fuente: {hist_data.get('fuente', 'Banxico SIE API · Histórico')}"
|
| 682 |
+
)
|
| 683 |
+
except Exception:
|
| 684 |
+
historico = historico_raw[:1500]
|
| 685 |
+
|
| 686 |
+
msgs = [
|
| 687 |
+
{"role": "system", "content": SYSTEM_STRICT},
|
| 688 |
+
{"role": "user", "content":
|
| 689 |
+
f"DATOS HISTÓRICOS EXACTOS:\n{historico}\n\n"
|
| 690 |
+
f"USA ESTOS DATOS para responder.\n"
|
| 691 |
+
f"PREGUNTA: {user_question}"
|
| 692 |
+
}
|
| 693 |
+
]
|
| 694 |
+
answer, error = call_model_groq(MODEL_NUMERICAL, msgs, max_tokens=700)
|
| 695 |
+
if error:
|
| 696 |
+
return f"⚠️ Error numérico: {error}"
|
| 697 |
+
|
| 698 |
+
if stream_placeholder:
|
| 699 |
+
stream_placeholder.markdown("*Validando datos...*")
|
| 700 |
+
validated = validate_response(answer, historico[:500], user_question)
|
| 701 |
+
if stream_placeholder:
|
| 702 |
+
stream_placeholder.markdown(validated)
|
| 703 |
+
return validated
|
| 704 |
+
|
| 705 |
+
# ── PASO 2C: NARRATIVA multi-fuente → paralelo + LLaMA 70B ───────────────
|
| 706 |
+
if needs_multi_source(user_question):
|
| 707 |
+
try:
|
| 708 |
+
loop = asyncio.get_event_loop()
|
| 709 |
+
if loop.is_closed():
|
| 710 |
+
loop = asyncio.new_event_loop()
|
| 711 |
+
asyncio.set_event_loop(loop)
|
| 712 |
+
context = loop.run_until_complete(prefetch_all_parallel(stream_placeholder))
|
| 713 |
+
except Exception:
|
| 714 |
+
parts = []
|
| 715 |
+
for tool, args in [("read_banxico", {"indicador": "todos"}),
|
| 716 |
+
("read_worldbank", {"categoria": "economia"}),
|
| 717 |
+
("read_inegi", {"indicador": "todos"})]:
|
| 718 |
+
parts.append(execute_tool(tool, args)[:500])
|
| 719 |
+
context = "\n".join(parts)
|
| 720 |
+
|
| 721 |
+
trace.set_route("NARRATIVA")
|
| 722 |
+
if stream_placeholder:
|
| 723 |
+
stream_placeholder.markdown("*Analizando con LLaMA 70B...*")
|
| 724 |
+
|
| 725 |
+
msgs = [
|
| 726 |
+
{"role": "system", "content": SYSTEM_STRICT},
|
| 727 |
+
{"role": "user", "content":
|
| 728 |
+
f"DATOS REALES DE LAS 3 FUENTES:\n{context}\n\n"
|
| 729 |
+
f"PREGUNTA: {user_question}"
|
| 730 |
+
}
|
| 731 |
+
]
|
| 732 |
+
narrative, error = call_model(MODEL_NARRATIVE, msgs, max_tokens=700)
|
| 733 |
+
if error:
|
| 734 |
+
narrative = context
|
| 735 |
+
|
| 736 |
+
final = synthesize_response(context, user_question, stream_placeholder)
|
| 737 |
+
if stream_placeholder:
|
| 738 |
+
stream_placeholder.markdown("*Validando respuesta...*")
|
| 739 |
+
validated = validate_response(final, context[:500], user_question)
|
| 740 |
+
if stream_placeholder:
|
| 741 |
+
stream_placeholder.markdown(validated)
|
| 742 |
+
return validated
|
| 743 |
+
|
| 744 |
+
# ── PASO 2D: NARRATIVA mono-fuente → tool calling con LLaMA 70B ──────────
|
| 745 |
+
headers = {
|
| 746 |
+
"Authorization": f"Bearer {NVIDIA_API_KEY}",
|
| 747 |
+
"Content-Type": "application/json",
|
| 748 |
+
}
|
| 749 |
+
msgs_trimmed = messages[-4:] if len(messages) > 4 else messages
|
| 750 |
+
current_messages = [{"role": "system", "content": SYSTEM_STRICT}] + msgs_trimmed
|
| 751 |
+
tool_results = []
|
| 752 |
+
|
| 753 |
+
for iteration in range(5):
|
| 754 |
+
payload = {
|
| 755 |
+
"model": MODEL_NARRATIVE,
|
| 756 |
+
"messages": current_messages,
|
| 757 |
+
"tools": TOOLS,
|
| 758 |
+
"tool_choice": "auto",
|
| 759 |
+
"temperature": 0.1,
|
| 760 |
+
"max_tokens": 700,
|
| 761 |
+
"stream": False,
|
| 762 |
+
}
|
| 763 |
+
if stream_placeholder:
|
| 764 |
+
stream_placeholder.markdown(f"*LLaMA 70B consultando... (iter {iteration+1})*")
|
| 765 |
+
|
| 766 |
+
try:
|
| 767 |
+
r = requests.post(
|
| 768 |
+
f"{NVIDIA_BASE_URL}/chat/completions",
|
| 769 |
+
headers=headers,
|
| 770 |
+
json=payload,
|
| 771 |
+
timeout=180,
|
| 772 |
+
)
|
| 773 |
+
r.raise_for_status()
|
| 774 |
+
response = r.json()
|
| 775 |
+
except requests.exceptions.HTTPError as e:
|
| 776 |
+
if "400" in str(e) and tool_results:
|
| 777 |
+
fallback = synthesize_response("\n".join(tool_results), user_question, stream_placeholder)
|
| 778 |
+
return validate_response(fallback, "\n".join(tool_results[:2]))
|
| 779 |
+
return f"⚠️ Error HTTP: {e}"
|
| 780 |
+
except Exception as e:
|
| 781 |
+
return f"⚠️ Error: {e}"
|
| 782 |
+
|
| 783 |
+
choice = response["choices"][0]
|
| 784 |
+
finish_reason = choice.get("finish_reason", "")
|
| 785 |
+
message = choice["message"]
|
| 786 |
+
|
| 787 |
+
if finish_reason == "stop" or not message.get("tool_calls"):
|
| 788 |
+
answer = message.get("content", "Sin respuesta.")
|
| 789 |
+
validated = validate_response(answer, "\n".join(tool_results[:2]) if tool_results else "", user_question)
|
| 790 |
+
if stream_placeholder:
|
| 791 |
+
stream_placeholder.markdown(validated)
|
| 792 |
+
return validated
|
| 793 |
+
|
| 794 |
+
tool_calls = message.get("tool_calls", [])
|
| 795 |
+
current_messages.append({
|
| 796 |
+
"role": "assistant",
|
| 797 |
+
"content": message.get("content", ""),
|
| 798 |
+
"tool_calls": tool_calls,
|
| 799 |
+
})
|
| 800 |
+
|
| 801 |
+
for tc in tool_calls:
|
| 802 |
+
tool_name = tc["function"]["name"]
|
| 803 |
+
try:
|
| 804 |
+
tool_args = json.loads(tc["function"]["arguments"])
|
| 805 |
+
except Exception:
|
| 806 |
+
tool_args = {}
|
| 807 |
+
if stream_placeholder:
|
| 808 |
+
stream_placeholder.markdown(f"*Ejecutando: {tool_name}...*")
|
| 809 |
+
if tool_name == "search_historico" and "query" not in tool_args:
|
| 810 |
+
tool_args["query"] = user_question
|
| 811 |
+
result = execute_tool(tool_name, tool_args)
|
| 812 |
+
if len(result) > 1500:
|
| 813 |
+
result = result[:1500] + " [truncado]"
|
| 814 |
+
tool_results.append(f"[{tool_name}]: {result[:500]}")
|
| 815 |
+
current_messages.append({
|
| 816 |
+
"role": "tool",
|
| 817 |
+
"tool_call_id": tc["id"],
|
| 818 |
+
"name": tool_name,
|
| 819 |
+
"content": result,
|
| 820 |
+
})
|
| 821 |
+
|
| 822 |
+
return "⚠️ Se alcanzó el límite de iteraciones."
|
| 823 |
+
|
| 824 |
+
|
| 825 |
+
def run_agent(user_question: str, history: list = None, stream_placeholder=None) -> str:
|
| 826 |
+
if history is None:
|
| 827 |
+
history = []
|
| 828 |
+
messages = history + [{"role": "user", "content": user_question}]
|
| 829 |
+
return call_llm_with_tools(messages, stream_placeholder)
|
| 830 |
+
|
| 831 |
+
|
| 832 |
+
if __name__ == "__main__":
|
| 833 |
+
preguntas = [
|
| 834 |
+
"¿Cuál es el tipo de cambio hoy?",
|
| 835 |
+
"¿Cuál fue el tipo de cambio en enero 2025?",
|
| 836 |
+
"¿Cuál es la participación de mercado de Telcel?",
|
| 837 |
+
"Dame un análisis económico completo de México cruzando Banxico, World Bank e INEGI",
|
| 838 |
+
]
|
| 839 |
+
for p in preguntas:
|
| 840 |
+
print(f"\nP: {p}")
|
| 841 |
+
print(f"R: {run_agent(p)}")
|
modules/agent_tools.py
ADDED
|
@@ -0,0 +1,764 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · Agent Tools
|
| 3 |
+
====================
|
| 4 |
+
Tools formales para el orquestador LLaMA 3.1 70B.
|
| 5 |
+
Cada tool lee directamente del bucket /data — cero alucinaciones.
|
| 6 |
+
|
| 7 |
+
Tools disponibles:
|
| 8 |
+
- read_banxico → /data/banxico.json
|
| 9 |
+
- search_historico → /data/banxico_historico.json
|
| 10 |
+
- read_worldbank → /data/worldbank.json
|
| 11 |
+
- read_inegi → /data/inegi.json
|
| 12 |
+
- search_wikipedia → Wikipedia API
|
| 13 |
+
- read_ift → /data/ift.json ← NUEVO
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
from datetime import datetime
|
| 19 |
+
|
| 20 |
+
DATA_PATH = "/data"
|
| 21 |
+
|
| 22 |
+
# ─── JSON Schema de Tools (para LLaMA tool calling) ─────────────────────────
|
| 23 |
+
|
| 24 |
+
TOOLS = [
|
| 25 |
+
{
|
| 26 |
+
"type": "function",
|
| 27 |
+
"function": {
|
| 28 |
+
"name": "search_web",
|
| 29 |
+
"description": "Búsqueda web en tiempo real via Tavily. Úsala cuando el dato no está en el bucket: portabilidad reciente, noticias CRT/ATDT, precios actuales, eventos regulatorios 2025-2026.",
|
| 30 |
+
"parameters": {
|
| 31 |
+
"type": "object",
|
| 32 |
+
"properties": {
|
| 33 |
+
"query": {"type": "string", "description": "Términos de búsqueda en español, incluye año y contexto México"},
|
| 34 |
+
"max_results": {"type": "integer", "description": "Resultados a retornar (default 3, max 5)"}
|
| 35 |
+
},
|
| 36 |
+
"required": ["query"]
|
| 37 |
+
}
|
| 38 |
+
}
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"type": "function",
|
| 42 |
+
"function": {
|
| 43 |
+
"name": "read_banxico",
|
| 44 |
+
"description": (
|
| 45 |
+
"Lee /data/banxico.json del bucket HF y devuelve datos exactos "
|
| 46 |
+
"de Banxico. Úsala para preguntas sobre tipo de cambio USD/MXN, "
|
| 47 |
+
"tasa TIIE, tasa objetivo del Banxico, o inflación IPC. "
|
| 48 |
+
"NUNCA inventes estos valores — siempre usa esta tool."
|
| 49 |
+
),
|
| 50 |
+
"parameters": {
|
| 51 |
+
"type": "object",
|
| 52 |
+
"properties": {
|
| 53 |
+
"indicador": {
|
| 54 |
+
"type": "string",
|
| 55 |
+
"enum": [
|
| 56 |
+
"tipo_cambio_usd_mxn",
|
| 57 |
+
"tiie_28_dias",
|
| 58 |
+
"tasa_objetivo",
|
| 59 |
+
"inflacion_ipc",
|
| 60 |
+
"todos"
|
| 61 |
+
],
|
| 62 |
+
"description": "Indicador específico a consultar. Usa 'todos' para resumen completo."
|
| 63 |
+
}
|
| 64 |
+
},
|
| 65 |
+
"required": ["indicador"]
|
| 66 |
+
}
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"type": "function",
|
| 71 |
+
"function": {
|
| 72 |
+
"name": "search_historico",
|
| 73 |
+
"description": (
|
| 74 |
+
"Busca en /data/banxico_historico.json series históricas 2024-2026. "
|
| 75 |
+
"Úsala para preguntas sobre tendencias, evolución en el tiempo, "
|
| 76 |
+
"comparaciones entre fechas, o máximos/mínimos históricos."
|
| 77 |
+
),
|
| 78 |
+
"parameters": {
|
| 79 |
+
"type": "object",
|
| 80 |
+
"properties": {
|
| 81 |
+
"serie": {
|
| 82 |
+
"type": "string",
|
| 83 |
+
"enum": ["SF43718", "SF43783", "SF61745", "SP1"],
|
| 84 |
+
"description": (
|
| 85 |
+
"ID de serie Banxico: "
|
| 86 |
+
"SF43718=tipo_cambio, SF43783=TIIE_28d, "
|
| 87 |
+
"SF61745=tasa_objetivo, SP1=inflacion"
|
| 88 |
+
)
|
| 89 |
+
},
|
| 90 |
+
"fecha_inicio": {
|
| 91 |
+
"type": "string",
|
| 92 |
+
"description": "Fecha inicio en formato DD/MM/YYYY. Opcional."
|
| 93 |
+
},
|
| 94 |
+
"fecha_fin": {
|
| 95 |
+
"type": "string",
|
| 96 |
+
"description": "Fecha fin en formato DD/MM/YYYY. Opcional."
|
| 97 |
+
},
|
| 98 |
+
"ultimos_n": {
|
| 99 |
+
"type": "integer",
|
| 100 |
+
"description": "Retornar los últimos N registros. Opcional."
|
| 101 |
+
},
|
| 102 |
+
"query": {
|
| 103 |
+
"type": "string",
|
| 104 |
+
"description": "Pregunta semántica para búsqueda en ChromaDB. Opcional."
|
| 105 |
+
}
|
| 106 |
+
},
|
| 107 |
+
"required": ["serie"]
|
| 108 |
+
}
|
| 109 |
+
}
|
| 110 |
+
},
|
| 111 |
+
{
|
| 112 |
+
"type": "function",
|
| 113 |
+
"function": {
|
| 114 |
+
"name": "read_worldbank",
|
| 115 |
+
"description": (
|
| 116 |
+
"Lee /data/worldbank.json con 22 indicadores de México del World Bank "
|
| 117 |
+
"en 6 categorías: economía, social, comercio, energía, medio ambiente, gobernanza. "
|
| 118 |
+
"Úsala para PIB, desempleo, pobreza, exportaciones, emisiones CO2, etc."
|
| 119 |
+
),
|
| 120 |
+
"parameters": {
|
| 121 |
+
"type": "object",
|
| 122 |
+
"properties": {
|
| 123 |
+
"categoria": {
|
| 124 |
+
"type": "string",
|
| 125 |
+
"enum": [
|
| 126 |
+
"economia", "social", "comercio",
|
| 127 |
+
"energia", "medio_ambiente", "gobernanza", "todos"
|
| 128 |
+
],
|
| 129 |
+
"description": "Categoría de indicadores. Usa 'todos' para resumen completo."
|
| 130 |
+
},
|
| 131 |
+
"indicador": {
|
| 132 |
+
"type": "string",
|
| 133 |
+
"description": "Indicador específico opcional."
|
| 134 |
+
}
|
| 135 |
+
},
|
| 136 |
+
"required": ["categoria"]
|
| 137 |
+
}
|
| 138 |
+
}
|
| 139 |
+
},
|
| 140 |
+
{
|
| 141 |
+
"type": "function",
|
| 142 |
+
"function": {
|
| 143 |
+
"name": "search_wikipedia",
|
| 144 |
+
"description": (
|
| 145 |
+
"Busca información conceptual y contextual en Wikipedia. "
|
| 146 |
+
"Úsala para preguntas sobre QUÉ ES algo, CÓMO FUNCIONA, HISTORIA o CONTEXTO. "
|
| 147 |
+
"NUNCA la uses para datos numéricos — para eso usa las otras tools."
|
| 148 |
+
),
|
| 149 |
+
"parameters": {
|
| 150 |
+
"type": "object",
|
| 151 |
+
"properties": {
|
| 152 |
+
"termino": {
|
| 153 |
+
"type": "string",
|
| 154 |
+
"description": "Término a buscar. Ej: 'TIIE México', 'Banco de México'."
|
| 155 |
+
},
|
| 156 |
+
"idioma": {
|
| 157 |
+
"type": "string",
|
| 158 |
+
"enum": ["es", "en"],
|
| 159 |
+
"description": "Idioma de búsqueda. Por defecto 'es'."
|
| 160 |
+
}
|
| 161 |
+
},
|
| 162 |
+
"required": ["termino"]
|
| 163 |
+
}
|
| 164 |
+
}
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"type": "function",
|
| 168 |
+
"function": {
|
| 169 |
+
"name": "read_inegi",
|
| 170 |
+
"description": (
|
| 171 |
+
"Lee /data/inegi.json con indicadores del INEGI en tiempo real. "
|
| 172 |
+
"Úsala para preguntas sobre población, IGAE (actividad económica), "
|
| 173 |
+
"ocupación/desempleo, índice de precios al consumidor."
|
| 174 |
+
),
|
| 175 |
+
"parameters": {
|
| 176 |
+
"type": "object",
|
| 177 |
+
"properties": {
|
| 178 |
+
"indicador": {
|
| 179 |
+
"type": "string",
|
| 180 |
+
"enum": ["poblacion", "igae", "ocupacion", "precios", "todos"],
|
| 181 |
+
"description": "Indicador INEGI a consultar."
|
| 182 |
+
}
|
| 183 |
+
},
|
| 184 |
+
"required": ["indicador"]
|
| 185 |
+
}
|
| 186 |
+
}
|
| 187 |
+
},
|
| 188 |
+
# ── NUEVA TOOL: IFT Telecomunicaciones ────────────────────────────────────
|
| 189 |
+
{
|
| 190 |
+
"type": "function",
|
| 191 |
+
"function": {
|
| 192 |
+
"name": "read_ift",
|
| 193 |
+
"description": (
|
| 194 |
+
"Lee /data/ift.json con datos del sector telecomunicaciones México (IFT). "
|
| 195 |
+
"Úsala para preguntas sobre: Telcel, AT&T, Movistar, Telmex, "
|
| 196 |
+
"participación de mercado móvil, ingresos telco, cobertura 4G/5G, "
|
| 197 |
+
"portabilidad numérica, TV restringida, regulación telco, "
|
| 198 |
+
"agente económico preponderante, banda ancha. "
|
| 199 |
+
"NUNCA inventes datos de telecomunicaciones — siempre usa esta tool."
|
| 200 |
+
),
|
| 201 |
+
"parameters": {
|
| 202 |
+
"type": "object",
|
| 203 |
+
"properties": {
|
| 204 |
+
"categoria": {
|
| 205 |
+
"type": "string",
|
| 206 |
+
"enum": [
|
| 207 |
+
"ingresos",
|
| 208 |
+
"participacion_mercado_movil",
|
| 209 |
+
"internet_fijo",
|
| 210 |
+
"indicadores_conectividad",
|
| 211 |
+
"television",
|
| 212 |
+
"portabilidad",
|
| 213 |
+
"cobertura_5g",
|
| 214 |
+
"regulacion",
|
| 215 |
+
"historico",
|
| 216 |
+
"todo"
|
| 217 |
+
],
|
| 218 |
+
"description": (
|
| 219 |
+
"Categoría de datos telco: "
|
| 220 |
+
"ingresos=Q1 2025 por operador, "
|
| 221 |
+
"participacion_mercado_movil=líneas por operador, "
|
| 222 |
+
"cobertura_5g=estado 5G/4G, "
|
| 223 |
+
"regulacion=AEP y contexto regulatorio, "
|
| 224 |
+
"todo=resumen ejecutivo completo."
|
| 225 |
+
)
|
| 226 |
+
}
|
| 227 |
+
},
|
| 228 |
+
"required": ["categoria"]
|
| 229 |
+
}
|
| 230 |
+
}
|
| 231 |
+
},
|
| 232 |
+
]
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
# ─── Implementación de cada tool ─────────────────────────────────────────────
|
| 236 |
+
|
| 237 |
+
def read_banxico(indicador: str) -> dict:
|
| 238 |
+
"""Lee banxico.json y retorna el dato solicitado."""
|
| 239 |
+
path = os.path.join(DATA_PATH, "banxico.json")
|
| 240 |
+
try:
|
| 241 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 242 |
+
data = json.load(f)
|
| 243 |
+
|
| 244 |
+
if indicador == "todos":
|
| 245 |
+
datos_fmt = {}
|
| 246 |
+
for k, v in data.items():
|
| 247 |
+
if k in ["fuente", "timestamp", "detalle", "actualizado"]:
|
| 248 |
+
continue
|
| 249 |
+
if isinstance(v, dict):
|
| 250 |
+
continue
|
| 251 |
+
try:
|
| 252 |
+
n = float(v)
|
| 253 |
+
if k == "inflacion_ipc" or k == "SP1":
|
| 254 |
+
datos_fmt[k] = f"{n:.3f} (índice base — NO usar como % de inflación)"
|
| 255 |
+
elif k in ["tipo_cambio_usd_mxn", "SF43718"]:
|
| 256 |
+
datos_fmt[k] = f"{n:.4f} MXN/USD"
|
| 257 |
+
elif k in ["tiie_28_dias", "SF43783", "tasa_objetivo", "SF61745"]:
|
| 258 |
+
datos_fmt[k] = f"{n:.4f}%"
|
| 259 |
+
else:
|
| 260 |
+
datos_fmt[k] = v
|
| 261 |
+
except (TypeError, ValueError):
|
| 262 |
+
datos_fmt[k] = v
|
| 263 |
+
return {
|
| 264 |
+
"fuente": "Banxico SIE API",
|
| 265 |
+
"archivo": "banxico.json",
|
| 266 |
+
"fecha_extraccion": data.get("timestamp", data.get("actualizado", "N/A")),
|
| 267 |
+
"nota_inflacion": "El campo inflacion_ipc es el ÍNDICE INPC base 2018=100, NO el porcentaje de inflación anual.",
|
| 268 |
+
"datos": datos_fmt
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
mapa = {
|
| 272 |
+
"tipo_cambio_usd_mxn": ["SF43718", "tipo_cambio", "tipo_cambio_usd_mxn"],
|
| 273 |
+
"tiie_28_dias": ["SF43783", "tiie_28_dias", "tiie"],
|
| 274 |
+
"tasa_objetivo": ["SF61745", "tasa_objetivo", "tasa_interes"],
|
| 275 |
+
"inflacion_ipc": ["SP1", "inflacion_ipc", "inflacion"],
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
claves = mapa.get(indicador, [indicador])
|
| 279 |
+
buscar_en = [data, data.get("datos", {}), data.get("detalle", {})]
|
| 280 |
+
for fuente_data in buscar_en:
|
| 281 |
+
if not isinstance(fuente_data, dict):
|
| 282 |
+
continue
|
| 283 |
+
for clave in claves:
|
| 284 |
+
if clave in fuente_data:
|
| 285 |
+
valor = fuente_data[clave]
|
| 286 |
+
if isinstance(valor, dict):
|
| 287 |
+
fecha = valor.get("fecha", data.get("timestamp", "N/A"))
|
| 288 |
+
valor = valor.get("valor", str(valor))
|
| 289 |
+
else:
|
| 290 |
+
fecha = data.get("timestamp", "N/A")
|
| 291 |
+
return {
|
| 292 |
+
"fuente": "Banxico SIE API",
|
| 293 |
+
"archivo": "banxico.json",
|
| 294 |
+
"indicador": indicador,
|
| 295 |
+
"valor": valor,
|
| 296 |
+
"fecha": fecha,
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
return {
|
| 300 |
+
"error": f"Indicador '{indicador}' no encontrado en banxico.json",
|
| 301 |
+
"claves_disponibles": list(data.keys()),
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
except FileNotFoundError:
|
| 305 |
+
return {"error": "banxico.json no encontrado en /data."}
|
| 306 |
+
except Exception as e:
|
| 307 |
+
return {"error": str(e)}
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def search_historico(serie: str, fecha_inicio: str = None,
|
| 311 |
+
fecha_fin: str = None, ultimos_n: int = None,
|
| 312 |
+
query: str = None) -> dict:
|
| 313 |
+
"""Busca en el histórico de Banxico. JSON directo para estadísticas."""
|
| 314 |
+
import re as _re
|
| 315 |
+
nombres = {
|
| 316 |
+
"SF43718": "Tipo de cambio USD/MXN",
|
| 317 |
+
"SF43783": "TIIE 28 días",
|
| 318 |
+
"SF61745": "Tasa objetivo Banxico",
|
| 319 |
+
"SP1": "Inflación IPC"
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
query_text = query or ""
|
| 323 |
+
years_in_query = _re.findall(r"\b(202[0-9])\b", query_text)
|
| 324 |
+
is_statistical = any(k in query_text.lower() for k in [
|
| 325 |
+
"promedio", "average", "mínimo", "minimo", "máximo", "maximo",
|
| 326 |
+
"estadísticas", "estadisticas", "statistics", "media", "mean"
|
| 327 |
+
])
|
| 328 |
+
use_json = years_in_query or is_statistical or not query_text
|
| 329 |
+
|
| 330 |
+
if not use_json:
|
| 331 |
+
try:
|
| 332 |
+
from modules.rag_chroma import search_historico_semantico
|
| 333 |
+
result = search_historico_semantico(query_text, serie=serie, n_results=20)
|
| 334 |
+
if result and "error" not in result and result.get("total", 0) > 0:
|
| 335 |
+
result["nombre"] = nombres.get(serie, serie)
|
| 336 |
+
result["tool"] = "ChromaDB · semántico"
|
| 337 |
+
result["resultados"] = result.get("resultados", [])[:5]
|
| 338 |
+
return result
|
| 339 |
+
except Exception:
|
| 340 |
+
pass
|
| 341 |
+
|
| 342 |
+
path = os.path.join(DATA_PATH, "banxico_historico.json")
|
| 343 |
+
try:
|
| 344 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 345 |
+
data = json.load(f)
|
| 346 |
+
|
| 347 |
+
registros = None
|
| 348 |
+
for clave in [serie, serie.upper(), serie.lower()]:
|
| 349 |
+
if clave in data:
|
| 350 |
+
raw = data[clave]
|
| 351 |
+
if isinstance(raw, list):
|
| 352 |
+
registros = raw
|
| 353 |
+
elif isinstance(raw, dict) and "registros" in raw:
|
| 354 |
+
registros = raw["registros"]
|
| 355 |
+
break
|
| 356 |
+
|
| 357 |
+
if registros is None:
|
| 358 |
+
return {"error": f"Serie '{serie}' no encontrada",
|
| 359 |
+
"series_disponibles": [k for k in data.keys() if k not in ["fuente","timestamp","periodo"]]}
|
| 360 |
+
|
| 361 |
+
if years_in_query:
|
| 362 |
+
filtrados = [r for r in registros
|
| 363 |
+
if any(yr in str(r.get("fecha","")) for yr in years_in_query)]
|
| 364 |
+
if filtrados:
|
| 365 |
+
registros = filtrados
|
| 366 |
+
|
| 367 |
+
if fecha_inicio or fecha_fin:
|
| 368 |
+
def parse_f(f):
|
| 369 |
+
for fmt in ["%d/%m/%Y", "%Y-%m-%d"]:
|
| 370 |
+
try:
|
| 371 |
+
return datetime.strptime(f, fmt)
|
| 372 |
+
except Exception:
|
| 373 |
+
pass
|
| 374 |
+
return None
|
| 375 |
+
fi = parse_f(fecha_inicio) if fecha_inicio else None
|
| 376 |
+
ff = parse_f(fecha_fin) if fecha_fin else None
|
| 377 |
+
filtrados2 = []
|
| 378 |
+
for r in registros:
|
| 379 |
+
fr = parse_f(r.get("fecha",""))
|
| 380 |
+
if fr:
|
| 381 |
+
if fi and fr < fi: continue
|
| 382 |
+
if ff and fr > ff: continue
|
| 383 |
+
filtrados2.append(r)
|
| 384 |
+
if filtrados2:
|
| 385 |
+
registros = filtrados2
|
| 386 |
+
|
| 387 |
+
if ultimos_n:
|
| 388 |
+
registros = registros[-ultimos_n:]
|
| 389 |
+
|
| 390 |
+
valores = []
|
| 391 |
+
for r in registros:
|
| 392 |
+
try:
|
| 393 |
+
v = str(r.get("dato", r.get("valor", ""))).split(" ")[0].replace(",","")
|
| 394 |
+
valores.append(float(v))
|
| 395 |
+
except Exception:
|
| 396 |
+
pass
|
| 397 |
+
|
| 398 |
+
stats = {}
|
| 399 |
+
if valores:
|
| 400 |
+
stats = {
|
| 401 |
+
"promedio": round(sum(valores)/len(valores), 4),
|
| 402 |
+
"minimo": round(min(valores), 4),
|
| 403 |
+
"maximo": round(max(valores), 4),
|
| 404 |
+
"primero": valores[0],
|
| 405 |
+
"ultimo": valores[-1],
|
| 406 |
+
}
|
| 407 |
+
|
| 408 |
+
return {
|
| 409 |
+
"fuente": "Banxico SIE API · Histórico · JSON directo",
|
| 410 |
+
"serie": serie,
|
| 411 |
+
"nombre": nombres.get(serie, serie),
|
| 412 |
+
"años_filtrados": years_in_query or "todos",
|
| 413 |
+
"total_registros": len(registros),
|
| 414 |
+
"estadisticas": stats,
|
| 415 |
+
"muestra": registros[-3:] if len(registros) > 3 else registros
|
| 416 |
+
}
|
| 417 |
+
|
| 418 |
+
except FileNotFoundError:
|
| 419 |
+
return {"error": "banxico_historico.json no encontrado en /data."}
|
| 420 |
+
except Exception as e:
|
| 421 |
+
return {"error": str(e)}
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def read_worldbank(categoria: str, indicador: str = None) -> dict:
|
| 425 |
+
"""Lee worldbank.json."""
|
| 426 |
+
path = os.path.join(DATA_PATH, "worldbank.json")
|
| 427 |
+
try:
|
| 428 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 429 |
+
data = json.load(f)
|
| 430 |
+
|
| 431 |
+
def fmt_num(v, key=""):
|
| 432 |
+
try:
|
| 433 |
+
n = float(v)
|
| 434 |
+
if key in ["pib_usd", "exportaciones_usd", "importaciones_usd",
|
| 435 |
+
"remesas_usd", "cuenta_corriente_usd"]:
|
| 436 |
+
if abs(n) > 1_000_000_000_000:
|
| 437 |
+
return f"${n/1_000_000_000_000:,.3f} billones USD"
|
| 438 |
+
elif abs(n) > 1_000_000_000:
|
| 439 |
+
return f"${n/1_000_000_000:,.2f} mil millones USD"
|
| 440 |
+
elif n > 1_000_000_000:
|
| 441 |
+
return f"${n/1_000_000_000:,.2f} B USD"
|
| 442 |
+
elif n > 1_000_000:
|
| 443 |
+
return f"{n:,.0f}"
|
| 444 |
+
return v
|
| 445 |
+
except (TypeError, ValueError):
|
| 446 |
+
return v
|
| 447 |
+
|
| 448 |
+
if categoria == "todos":
|
| 449 |
+
return {
|
| 450 |
+
"fuente": "World Bank API",
|
| 451 |
+
"archivo": "worldbank.json",
|
| 452 |
+
"fecha_extraccion": data.get("timestamp", "N/A"),
|
| 453 |
+
"datos": data
|
| 454 |
+
}
|
| 455 |
+
|
| 456 |
+
resultado = None
|
| 457 |
+
for clave in [categoria, categoria.lower(), categoria.upper()]:
|
| 458 |
+
if clave in data:
|
| 459 |
+
resultado = data[clave]
|
| 460 |
+
break
|
| 461 |
+
|
| 462 |
+
if resultado is None:
|
| 463 |
+
todas = {k: v for k, v in data.items()
|
| 464 |
+
if k not in ["fuente", "timestamp", "pais"] and isinstance(v, dict)}
|
| 465 |
+
if todas:
|
| 466 |
+
return {
|
| 467 |
+
"fuente": "World Bank API",
|
| 468 |
+
"nota": f"Categoría '{categoria}' no encontrada — devolviendo todos los datos",
|
| 469 |
+
"datos": todas,
|
| 470 |
+
}
|
| 471 |
+
return {
|
| 472 |
+
"error": f"Categoría '{categoria}' no encontrada",
|
| 473 |
+
"categorias_disponibles": [k for k in data.keys() if k not in ["fuente","timestamp","pais"]]
|
| 474 |
+
}
|
| 475 |
+
|
| 476 |
+
if indicador and isinstance(resultado, dict):
|
| 477 |
+
for k, v in resultado.items():
|
| 478 |
+
if indicador.lower() in k.lower():
|
| 479 |
+
return {
|
| 480 |
+
"fuente": "World Bank API",
|
| 481 |
+
"categoria": categoria,
|
| 482 |
+
"indicador": k,
|
| 483 |
+
"valor": v
|
| 484 |
+
}
|
| 485 |
+
|
| 486 |
+
if isinstance(resultado, dict):
|
| 487 |
+
for k, v in resultado.items():
|
| 488 |
+
if isinstance(v, dict) and "valor" in v:
|
| 489 |
+
v["valor_formateado"] = fmt_num(v["valor"])
|
| 490 |
+
|
| 491 |
+
return {
|
| 492 |
+
"fuente": "World Bank API",
|
| 493 |
+
"archivo": "worldbank.json",
|
| 494 |
+
"categoria": categoria,
|
| 495 |
+
"datos": resultado
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
except FileNotFoundError:
|
| 499 |
+
return {"error": "worldbank.json no encontrado en /data."}
|
| 500 |
+
except Exception as e:
|
| 501 |
+
return {"error": str(e)}
|
| 502 |
+
|
| 503 |
+
|
| 504 |
+
def read_inegi(indicador: str) -> dict:
|
| 505 |
+
"""Lee inegi.json."""
|
| 506 |
+
path = os.path.join(DATA_PATH, "inegi.json")
|
| 507 |
+
try:
|
| 508 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 509 |
+
data = json.load(f)
|
| 510 |
+
|
| 511 |
+
if indicador == "todos":
|
| 512 |
+
# Inyectar poblacion_total si no está (INEGI API falla frecuentemente)
|
| 513 |
+
if "poblacion_total" not in data:
|
| 514 |
+
data["poblacion_total"] = "128,455,567 personas (World Bank 2023 · SP.POP.TOTL)"
|
| 515 |
+
return {
|
| 516 |
+
"fuente": "INEGI · INEGIpy + World Bank complemento",
|
| 517 |
+
"archivo": "inegi.json",
|
| 518 |
+
"fecha_extraccion": data.get("timestamp", "N/A"),
|
| 519 |
+
"datos": data
|
| 520 |
+
}
|
| 521 |
+
|
| 522 |
+
mapa = {
|
| 523 |
+
"poblacion": ["poblacion_total", "poblacion", "population", "pob_total"],
|
| 524 |
+
"igae": ["igae_mensual", "igae", "IGAE", "actividad_economica"],
|
| 525 |
+
"ocupacion": ["poblacion_ocupada", "ocupacion", "tasa_ocupacion", "tasa_desocupacion", "desempleo"],
|
| 526 |
+
"precios": ["inpc", "precios", "INPC", "indice_precios"],
|
| 527 |
+
}
|
| 528 |
+
|
| 529 |
+
buscar_en = [data, data.get("detalle", {})]
|
| 530 |
+
claves = mapa.get(indicador, [indicador])
|
| 531 |
+
for fuente_data in buscar_en:
|
| 532 |
+
if not isinstance(fuente_data, dict):
|
| 533 |
+
continue
|
| 534 |
+
for clave in claves:
|
| 535 |
+
if clave in fuente_data:
|
| 536 |
+
valor = fuente_data[clave]
|
| 537 |
+
if isinstance(valor, dict):
|
| 538 |
+
valor = valor.get("valor", str(valor))
|
| 539 |
+
return {
|
| 540 |
+
"fuente": "INEGI · INEGIpy",
|
| 541 |
+
"archivo": "inegi.json",
|
| 542 |
+
"indicador": indicador,
|
| 543 |
+
"valor": valor,
|
| 544 |
+
"fecha_extraccion": data.get("timestamp", "N/A")
|
| 545 |
+
}
|
| 546 |
+
|
| 547 |
+
claves_utiles = {k: v for k, v in data.items()
|
| 548 |
+
if k not in ["fuente", "timestamp", "detalle"] and not isinstance(v, dict)}
|
| 549 |
+
if claves_utiles:
|
| 550 |
+
datos_fmt = {}
|
| 551 |
+
for k, v in claves_utiles.items():
|
| 552 |
+
try:
|
| 553 |
+
n = float(v)
|
| 554 |
+
datos_fmt[k] = f"{n:,.0f}" if n > 1_000_000 else v
|
| 555 |
+
except (TypeError, ValueError):
|
| 556 |
+
datos_fmt[k] = v
|
| 557 |
+
# Agregar poblacion_total si no está en INEGI (API falla frecuentemente)
|
| 558 |
+
if "poblacion_total" not in datos_fmt:
|
| 559 |
+
datos_fmt["poblacion_total"] = "128,455,567 personas (World Bank 2023 · SP.POP.TOTL · usado como complemento porque INEGI API no devolvió este dato)"
|
| 560 |
+
return {
|
| 561 |
+
"fuente": "INEGI · INEGIpy + World Bank complemento",
|
| 562 |
+
"archivo": "inegi.json",
|
| 563 |
+
"datos_disponibles": datos_fmt,
|
| 564 |
+
"fecha_extraccion": data.get("timestamp", "N/A")
|
| 565 |
+
}
|
| 566 |
+
# Fallback hardcoded para poblacion_total cuando INEGI API falla
|
| 567 |
+
return {
|
| 568 |
+
"fuente": "INEGI · INEGIpy + World Bank SP.POP.TOTL",
|
| 569 |
+
"datos_disponibles": {
|
| 570 |
+
"poblacion_total": "128,455,567 personas (World Bank 2023 · SP.POP.TOTL)",
|
| 571 |
+
"nota": "Dato INEGI API no disponible temporalmente — usando World Bank"
|
| 572 |
+
},
|
| 573 |
+
"fecha_extraccion": "2023"
|
| 574 |
+
}
|
| 575 |
+
|
| 576 |
+
except FileNotFoundError:
|
| 577 |
+
return {"error": "inegi.json no encontrado en /data."}
|
| 578 |
+
except Exception as e:
|
| 579 |
+
return {"error": str(e)}
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def search_wikipedia(termino: str, idioma: str = "es") -> dict:
|
| 583 |
+
"""Busca información conceptual en Wikipedia."""
|
| 584 |
+
import requests as _req
|
| 585 |
+
try:
|
| 586 |
+
url = f"https://{idioma}.wikipedia.org/api/rest_v1/page/summary/{termino.replace(' ', '_')}"
|
| 587 |
+
r = _req.get(url, timeout=10, headers={"User-Agent": "MXDataIntelligence/1.0"})
|
| 588 |
+
|
| 589 |
+
if r.status_code == 404:
|
| 590 |
+
search_url = f"https://{idioma}.wikipedia.org/w/api.php"
|
| 591 |
+
params = {"action": "opensearch", "search": termino, "limit": 3, "format": "json"}
|
| 592 |
+
rs = _req.get(search_url, params=params, timeout=10)
|
| 593 |
+
if rs.status_code == 200:
|
| 594 |
+
results = rs.json()
|
| 595 |
+
if results[1]:
|
| 596 |
+
primer = results[1][0].replace(" ", "_")
|
| 597 |
+
r2 = _req.get(
|
| 598 |
+
f"https://{idioma}.wikipedia.org/api/rest_v1/page/summary/{primer}",
|
| 599 |
+
timeout=10, headers={"User-Agent": "MXDataIntelligence/1.0"}
|
| 600 |
+
)
|
| 601 |
+
if r2.status_code == 200:
|
| 602 |
+
data = r2.json()
|
| 603 |
+
return {
|
| 604 |
+
"fuente": f"Wikipedia ({idioma})",
|
| 605 |
+
"termino": termino,
|
| 606 |
+
"titulo": data.get("title", ""),
|
| 607 |
+
"resumen": data.get("extract", "")[:800],
|
| 608 |
+
"url": data.get("content_urls", {}).get("desktop", {}).get("page", ""),
|
| 609 |
+
}
|
| 610 |
+
return {"error": f"No se encontró '{termino}' en Wikipedia ({idioma})"}
|
| 611 |
+
|
| 612 |
+
r.raise_for_status()
|
| 613 |
+
data = r.json()
|
| 614 |
+
return {
|
| 615 |
+
"fuente": f"Wikipedia ({idioma})",
|
| 616 |
+
"termino": termino,
|
| 617 |
+
"titulo": data.get("title", ""),
|
| 618 |
+
"resumen": data.get("extract", "")[:800],
|
| 619 |
+
"url": data.get("content_urls", {}).get("desktop", {}).get("page", ""),
|
| 620 |
+
}
|
| 621 |
+
|
| 622 |
+
except Exception as e:
|
| 623 |
+
return {"error": f"Error consultando Wikipedia: {str(e)}"}
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
# ── NUEVA FUNCIÓN: read_ift ───────────────────────────────────────────────────
|
| 627 |
+
|
| 628 |
+
def read_ift(categoria: str = "todo") -> dict:
|
| 629 |
+
"""Lee /data/ift.json con datos del sector telecomunicaciones México."""
|
| 630 |
+
path = os.path.join(DATA_PATH, "ift.json")
|
| 631 |
+
try:
|
| 632 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 633 |
+
data = json.load(f)
|
| 634 |
+
|
| 635 |
+
CATEGORIA_MAP = {
|
| 636 |
+
"ingresos": "ingresos_q1_2025",
|
| 637 |
+
"participacion_mercado_movil": "participacion_mercado_movil",
|
| 638 |
+
"internet_fijo": "internet_fijo_q1_2025",
|
| 639 |
+
"indicadores_conectividad": "indicadores_conectividad",
|
| 640 |
+
"television": "television_restringida_q1_2025",
|
| 641 |
+
"portabilidad": "portabilidad",
|
| 642 |
+
"cobertura_5g": "cobertura_5g",
|
| 643 |
+
"regulacion": "contexto_regulatorio",
|
| 644 |
+
"historico": "serie_historica_ingresos_sector",
|
| 645 |
+
"todo": None,
|
| 646 |
+
}
|
| 647 |
+
|
| 648 |
+
if categoria == "todo":
|
| 649 |
+
return {
|
| 650 |
+
"fuente": data.get("metadatos", {}).get("fuente", "IFT"),
|
| 651 |
+
"periodo": data.get("metadatos", {}).get("periodo", "Q1 2025"),
|
| 652 |
+
"nota": data.get("metadatos", {}).get("nota", ""),
|
| 653 |
+
"resumen": {
|
| 654 |
+
"ingresos_sector_q1_2025_millones_mxn":
|
| 655 |
+
data.get("ingresos_q1_2025", {}).get("total_sector_millones_mxn"),
|
| 656 |
+
"top_operadores_ingresos": [
|
| 657 |
+
{k: v for k, v in op.items()
|
| 658 |
+
if k in ("marca", "ingreso_millones_mxn", "participacion_pct")}
|
| 659 |
+
for op in data.get("ingresos_q1_2025", {}).get("operadores", [])[:5]
|
| 660 |
+
],
|
| 661 |
+
"participacion_mercado_movil":
|
| 662 |
+
data.get("participacion_mercado_movil"),
|
| 663 |
+
"indicadores_conectividad":
|
| 664 |
+
data.get("indicadores_conectividad"),
|
| 665 |
+
"cobertura_5g":
|
| 666 |
+
data.get("cobertura_5g"),
|
| 667 |
+
"timestamp": data.get("actualizacion", {}).get("timestamp"),
|
| 668 |
+
}
|
| 669 |
+
}
|
| 670 |
+
|
| 671 |
+
key = CATEGORIA_MAP.get(categoria)
|
| 672 |
+
if key and key in data:
|
| 673 |
+
return {
|
| 674 |
+
"fuente": data.get("metadatos", {}).get("fuente", "IFT"),
|
| 675 |
+
"periodo": data.get("metadatos", {}).get("periodo", "Q1 2025"),
|
| 676 |
+
"categoria": categoria,
|
| 677 |
+
"datos": data[key],
|
| 678 |
+
}
|
| 679 |
+
|
| 680 |
+
return {
|
| 681 |
+
"error": f"Categoría '{categoria}' no encontrada",
|
| 682 |
+
"categorias_disponibles": list(CATEGORIA_MAP.keys()),
|
| 683 |
+
}
|
| 684 |
+
|
| 685 |
+
except FileNotFoundError:
|
| 686 |
+
return {
|
| 687 |
+
"error": "ift.json no encontrado en /data. Ejecuta data_extractor.py primero.",
|
| 688 |
+
"sugerencia": "Ve a Administración → Actualizar datos · Capa 2 en la UI."
|
| 689 |
+
}
|
| 690 |
+
except Exception as e:
|
| 691 |
+
return {"error": str(e)}
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
# ─── Dispatcher — ejecuta la tool por nombre ─────────────────────────────────
|
| 695 |
+
|
| 696 |
+
def search_web_tool(query: str = "", max_results: int = 3, **kwargs) -> str:
|
| 697 |
+
"""Búsqueda web en tiempo real via Tavily API."""
|
| 698 |
+
import os, requests as _req
|
| 699 |
+
api_key = os.getenv("TAVILY_API_KEY", "")
|
| 700 |
+
if not api_key:
|
| 701 |
+
return json.dumps({"error": "TAVILY_API_KEY no configurada — agrega el secret en HF Space Settings"})
|
| 702 |
+
if not query:
|
| 703 |
+
return json.dumps({"error": "query vacía"})
|
| 704 |
+
try:
|
| 705 |
+
payload = {
|
| 706 |
+
"api_key": api_key,
|
| 707 |
+
"query": query,
|
| 708 |
+
"max_results": min(max_results, 5),
|
| 709 |
+
"search_depth": "basic",
|
| 710 |
+
"include_answer": True,
|
| 711 |
+
"include_domains": [
|
| 712 |
+
"ift.org.mx", "crt.gob.mx", "atdt.gob.mx",
|
| 713 |
+
"banxico.org.mx", "inegi.org.mx",
|
| 714 |
+
"eleconomista.com.mx", "expansion.mx",
|
| 715 |
+
"milenio.com", "elfinanciero.com.mx",
|
| 716 |
+
"eluniversal.com.mx", "reforma.com"
|
| 717 |
+
]
|
| 718 |
+
}
|
| 719 |
+
r = _req.post("https://api.tavily.com/search", json=payload, timeout=15)
|
| 720 |
+
r.raise_for_status()
|
| 721 |
+
data = r.json()
|
| 722 |
+
results = []
|
| 723 |
+
for item in data.get("results", [])[:max_results]:
|
| 724 |
+
results.append({
|
| 725 |
+
"titulo": item.get("title", ""),
|
| 726 |
+
"url": item.get("url", ""),
|
| 727 |
+
"snippet": item.get("content", "")[:500],
|
| 728 |
+
"fecha": item.get("published_date", "reciente")
|
| 729 |
+
})
|
| 730 |
+
return json.dumps({
|
| 731 |
+
"answer": data.get("answer", ""),
|
| 732 |
+
"results": results,
|
| 733 |
+
"query": query,
|
| 734 |
+
"fuente": "Tavily Web Search · tiempo real"
|
| 735 |
+
}, ensure_ascii=False)
|
| 736 |
+
except Exception as e:
|
| 737 |
+
return json.dumps({"error": str(e)})
|
| 738 |
+
|
| 739 |
+
|
| 740 |
+
def execute_tool(tool_name: str, tool_args: dict) -> str:
|
| 741 |
+
"""Recibe nombre y argumentos de la tool call y retorna resultado como JSON string."""
|
| 742 |
+
REGISTRY = {
|
| 743 |
+
"search_web": search_web_tool,
|
| 744 |
+
"read_banxico": read_banxico,
|
| 745 |
+
"search_historico": search_historico,
|
| 746 |
+
"read_worldbank": read_worldbank,
|
| 747 |
+
"read_inegi": read_inegi,
|
| 748 |
+
"search_wikipedia": search_wikipedia,
|
| 749 |
+
"read_ift": read_ift, # ← NUEVO
|
| 750 |
+
}
|
| 751 |
+
|
| 752 |
+
fn = REGISTRY.get(tool_name)
|
| 753 |
+
if fn is None:
|
| 754 |
+
return json.dumps({
|
| 755 |
+
"error": f"Tool '{tool_name}' no existe. Tools disponibles: {list(REGISTRY.keys())}"
|
| 756 |
+
})
|
| 757 |
+
|
| 758 |
+
try:
|
| 759 |
+
result = fn(**tool_args)
|
| 760 |
+
return json.dumps(result, ensure_ascii=False, indent=2)
|
| 761 |
+
except TypeError as e:
|
| 762 |
+
return json.dumps({"error": f"Argumentos inválidos para '{tool_name}': {str(e)}"})
|
| 763 |
+
except Exception as e:
|
| 764 |
+
return json.dumps({"error": str(e)})
|
modules/critic_loop.py
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · Critic-Loop — Agente Evaluador Matemático
|
| 3 |
+
====================================================
|
| 4 |
+
Implementa el pipeline: Generador → Evaluador → Corrección → Respuesta Final
|
| 5 |
+
|
| 6 |
+
Arquitectura:
|
| 7 |
+
[Respuesta borrador del LLM]
|
| 8 |
+
↓
|
| 9 |
+
[EVALUADOR Python — verificación matemática instantánea]
|
| 10 |
+
↓
|
| 11 |
+
[¿Hay errores?] — Sí → [LLaMA 8B reescribe con feedback específico]
|
| 12 |
+
↓ No
|
| 13 |
+
[Respuesta Final]
|
| 14 |
+
|
| 15 |
+
El evaluador Python es determinístico — no usa LLM para detectar errores numéricos.
|
| 16 |
+
El LLM solo se usa para REESCRIBIR si hay error detectado.
|
| 17 |
+
Máximo 2 iteraciones para evitar loops infinitos.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import re
|
| 21 |
+
import json
|
| 22 |
+
import os
|
| 23 |
+
from datetime import datetime
|
| 24 |
+
|
| 25 |
+
DATA_PATH = "/data"
|
| 26 |
+
ERROR_LOG = os.path.join(DATA_PATH, "error_catalog.json")
|
| 27 |
+
MAX_ITER = 2 # máximo de reintentos
|
| 28 |
+
|
| 29 |
+
# ─── Catálogo de errores conocidos ───────────────────────────────────────────
|
| 30 |
+
# Se carga al inicio y se actualiza cuando el evaluador detecta errores nuevos.
|
| 31 |
+
|
| 32 |
+
def load_error_catalog() -> dict:
|
| 33 |
+
"""Carga el catálogo de errores conocidos desde /data/error_catalog.json."""
|
| 34 |
+
try:
|
| 35 |
+
if os.path.exists(ERROR_LOG):
|
| 36 |
+
with open(ERROR_LOG, "r", encoding="utf-8") as f:
|
| 37 |
+
return json.load(f)
|
| 38 |
+
except Exception:
|
| 39 |
+
pass
|
| 40 |
+
return {
|
| 41 |
+
"version": "1.0",
|
| 42 |
+
"actualizado": datetime.now().isoformat(),
|
| 43 |
+
"reglas_escala": [
|
| 44 |
+
{
|
| 45 |
+
"id": "SCALE_001",
|
| 46 |
+
"patron": r"\b(\d+[\.,]?\d*)\s*billones?\s+USD\b",
|
| 47 |
+
"descripcion": "billion en inglés = mil millones en español, NO billón",
|
| 48 |
+
"ejemplo_mal": "681.35 billones USD",
|
| 49 |
+
"ejemplo_bien": "681 mil millones USD",
|
| 50 |
+
"detectado": 0,
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"id": "SCALE_002",
|
| 54 |
+
"patron": r"inflaci[oó]n[^.]*?(\d+[\.,]\d+)\s*%[^.]*?[ií]ndice",
|
| 55 |
+
"descripcion": "INPC es índice base 2018=100, NO porcentaje de inflación",
|
| 56 |
+
"ejemplo_mal": "inflación de 127.5%",
|
| 57 |
+
"ejemplo_bien": "INPC índice: 127.5 (base 2018=100)",
|
| 58 |
+
"detectado": 0,
|
| 59 |
+
},
|
| 60 |
+
],
|
| 61 |
+
"errores_detectados": [],
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def save_error_catalog(catalog: dict):
|
| 66 |
+
"""Guarda el catálogo actualizado."""
|
| 67 |
+
try:
|
| 68 |
+
os.makedirs(DATA_PATH, exist_ok=True)
|
| 69 |
+
catalog["actualizado"] = datetime.now().isoformat()
|
| 70 |
+
with open(ERROR_LOG, "w", encoding="utf-8") as f:
|
| 71 |
+
json.dump(catalog, f, ensure_ascii=False, indent=2)
|
| 72 |
+
except Exception:
|
| 73 |
+
pass
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ─── Verificadores matemáticos Python (determinísticos) ─────────────────────
|
| 77 |
+
|
| 78 |
+
def _check_scale_billions(text: str, raw_data: dict) -> list:
|
| 79 |
+
"""
|
| 80 |
+
Verifica que los valores USD del World Bank estén en la escala correcta.
|
| 81 |
+
Un 'billion' USD = 1,000 millones USD (NO 1 billón en español).
|
| 82 |
+
"""
|
| 83 |
+
errors = []
|
| 84 |
+
|
| 85 |
+
# Patrones de error: número seguido de "billones USD" cuando debería ser "mil millones"
|
| 86 |
+
patron_error = re.compile(
|
| 87 |
+
r'(\d{1,4}(?:[\.,]\d+)?)\s*billones?\s+(?:de\s+)?USD',
|
| 88 |
+
re.IGNORECASE
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
for match in patron_error.finditer(text):
|
| 92 |
+
valor_str = match.group(1).replace(',', '.')
|
| 93 |
+
try:
|
| 94 |
+
valor = float(valor_str)
|
| 95 |
+
# Si el valor está entre 1 y 9999, probablemente es "miles de millones"
|
| 96 |
+
# mal expresado como "billones"
|
| 97 |
+
if 1 <= valor <= 9999:
|
| 98 |
+
# Verificar contra datos reales si están disponibles
|
| 99 |
+
context = text[max(0, match.start()-50):match.end()+50]
|
| 100 |
+
errors.append({
|
| 101 |
+
"tipo": "SCALE_001",
|
| 102 |
+
"texto": match.group(0),
|
| 103 |
+
"valor": valor,
|
| 104 |
+
"contexto": context.strip(),
|
| 105 |
+
"feedback": (
|
| 106 |
+
f"ERROR DE ESCALA: '{match.group(0)}' usa 'billones' incorrectamente. "
|
| 107 |
+
f"En español mexicano, {valor:.0f} billion USD = "
|
| 108 |
+
f"{valor:.0f} mil millones USD (NO {valor:.0f} billones). "
|
| 109 |
+
f"Reescribe usando 'mil millones USD'."
|
| 110 |
+
)
|
| 111 |
+
})
|
| 112 |
+
except ValueError:
|
| 113 |
+
pass
|
| 114 |
+
|
| 115 |
+
return errors
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def _check_inpc_as_pct(text: str) -> list:
|
| 119 |
+
"""
|
| 120 |
+
Verifica que el INPC no se presente como porcentaje de inflación.
|
| 121 |
+
El INPC es un índice base 2018=100, no un porcentaje.
|
| 122 |
+
"""
|
| 123 |
+
errors = []
|
| 124 |
+
|
| 125 |
+
# Detectar cuando se menciona inflación con valor > 50% (imposible — sería el índice)
|
| 126 |
+
patron = re.compile(
|
| 127 |
+
r'inflaci[oó]n[^.]{0,50}?(\d{2,3}[\.,]\d+)\s*%',
|
| 128 |
+
re.IGNORECASE
|
| 129 |
+
)
|
| 130 |
+
for match in patron.finditer(text):
|
| 131 |
+
try:
|
| 132 |
+
valor = float(match.group(1).replace(',', '.'))
|
| 133 |
+
if valor > 20: # inflación > 20% es sospechosa — probablemente es el índice
|
| 134 |
+
errors.append({
|
| 135 |
+
"tipo": "SCALE_002",
|
| 136 |
+
"texto": match.group(0),
|
| 137 |
+
"valor": valor,
|
| 138 |
+
"feedback": (
|
| 139 |
+
f"ERROR INPC: '{match.group(0)}' presenta el INPC como porcentaje. "
|
| 140 |
+
f"El valor {valor} es el ÍNDICE INPC base 2018=100, NO el porcentaje de inflación. "
|
| 141 |
+
f"Reescribe indicando que es el índice, no la tasa de inflación anual."
|
| 142 |
+
)
|
| 143 |
+
})
|
| 144 |
+
except ValueError:
|
| 145 |
+
pass
|
| 146 |
+
|
| 147 |
+
return errors
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _check_tiie_hallucination(text: str, raw_data: dict) -> list:
|
| 151 |
+
"""
|
| 152 |
+
Verifica que los valores de TIIE o tasa objetivo sean coherentes
|
| 153 |
+
con los datos del bucket.
|
| 154 |
+
"""
|
| 155 |
+
errors = []
|
| 156 |
+
|
| 157 |
+
# Obtener valor real de tasa del bucket
|
| 158 |
+
tasa_real = None
|
| 159 |
+
tiie_real = None
|
| 160 |
+
try:
|
| 161 |
+
banxico = raw_data.get("banxico", {})
|
| 162 |
+
tasa_real = float(banxico.get("tasa_objetivo", banxico.get("SF61745", 0)) or 0)
|
| 163 |
+
tiie_real = float(banxico.get("tiie_28_dias", banxico.get("SF43783", 0)) or 0)
|
| 164 |
+
except Exception:
|
| 165 |
+
pass
|
| 166 |
+
|
| 167 |
+
if not tasa_real:
|
| 168 |
+
return errors
|
| 169 |
+
|
| 170 |
+
# Buscar menciones de la tasa en el texto
|
| 171 |
+
patron_tasa = re.compile(
|
| 172 |
+
r'tasa\s+objetivo[^.]{0,30}?(\d+[\.,]\d+)\s*%',
|
| 173 |
+
re.IGNORECASE
|
| 174 |
+
)
|
| 175 |
+
for match in patron_tasa.finditer(text):
|
| 176 |
+
try:
|
| 177 |
+
valor_texto = float(match.group(1).replace(',', '.'))
|
| 178 |
+
# Tolerancia de 0.5% — diferencias mayores son sospechosas
|
| 179 |
+
if abs(valor_texto - tasa_real) > 0.5:
|
| 180 |
+
errors.append({
|
| 181 |
+
"tipo": "DATA_001",
|
| 182 |
+
"texto": match.group(0),
|
| 183 |
+
"valor": valor_texto,
|
| 184 |
+
"real": tasa_real,
|
| 185 |
+
"feedback": (
|
| 186 |
+
f"ERROR DATOS: La tasa objetivo mencionada es {valor_texto}% "
|
| 187 |
+
f"pero el dato real del bucket es {tasa_real}%. "
|
| 188 |
+
f"Corrige usando el valor exacto del bucket: {tasa_real}%."
|
| 189 |
+
)
|
| 190 |
+
})
|
| 191 |
+
except ValueError:
|
| 192 |
+
pass
|
| 193 |
+
|
| 194 |
+
return errors
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _check_math_operations(text: str) -> list:
|
| 198 |
+
"""
|
| 199 |
+
Verifica operaciones matemáticas simples expresadas en el texto.
|
| 200 |
+
Ej: "X representa el Y% de Z" → verifica que X/Z ≈ Y/100
|
| 201 |
+
"""
|
| 202 |
+
errors = []
|
| 203 |
+
|
| 204 |
+
# Patrón: "X millones MXN, lo que representa el Y% de Z millones"
|
| 205 |
+
patron = re.compile(
|
| 206 |
+
r'([\d,\.]+)\s*millones[^,]{0,60}?representa[^.]{0,60}?(\d+[\.,]\d+)\s*%[^.]{0,60}?([\d,\.]+)\s*millones',
|
| 207 |
+
re.IGNORECASE
|
| 208 |
+
)
|
| 209 |
+
for match in patron.finditer(text):
|
| 210 |
+
try:
|
| 211 |
+
parte = float(match.group(1).replace(',', '').replace('.', '')) if ',' in match.group(1) else float(match.group(1))
|
| 212 |
+
pct = float(match.group(2).replace(',', '.'))
|
| 213 |
+
total = float(match.group(3).replace(',', '').replace('.', '')) if ',' in match.group(3) else float(match.group(3))
|
| 214 |
+
if total > 0:
|
| 215 |
+
pct_calc = (parte / total) * 100
|
| 216 |
+
if abs(pct_calc - pct) > 1.0: # tolerancia 1%
|
| 217 |
+
errors.append({
|
| 218 |
+
"tipo": "MATH_001",
|
| 219 |
+
"texto": match.group(0),
|
| 220 |
+
"feedback": (
|
| 221 |
+
f"ERROR MATEMÁTICO: {parte:,.0f}/{total:,.0f} = {pct_calc:.1f}%, "
|
| 222 |
+
f"pero el texto dice {pct}%. Corrige el porcentaje."
|
| 223 |
+
)
|
| 224 |
+
})
|
| 225 |
+
except (ValueError, ZeroDivisionError):
|
| 226 |
+
pass
|
| 227 |
+
|
| 228 |
+
return errors
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# ─── Evaluador principal ─────────────────────────────────────────────────────
|
| 232 |
+
|
| 233 |
+
def evaluate_response(text: str, raw_data: dict = None) -> dict:
|
| 234 |
+
"""
|
| 235 |
+
Evaluador determinístico Python. Detecta errores sin llamar a LLM.
|
| 236 |
+
|
| 237 |
+
Args:
|
| 238 |
+
text: Respuesta borrador del LLM generador
|
| 239 |
+
raw_data: Dict con datos crudos del bucket para verificación cruzada
|
| 240 |
+
|
| 241 |
+
Returns:
|
| 242 |
+
{
|
| 243 |
+
"valid": bool,
|
| 244 |
+
"errors": [...],
|
| 245 |
+
"feedback": str # mensaje consolidado para el LLM corrector
|
| 246 |
+
}
|
| 247 |
+
"""
|
| 248 |
+
if raw_data is None:
|
| 249 |
+
raw_data = {}
|
| 250 |
+
|
| 251 |
+
all_errors = []
|
| 252 |
+
all_errors.extend(_check_scale_billions(text, raw_data))
|
| 253 |
+
all_errors.extend(_check_inpc_as_pct(text))
|
| 254 |
+
all_errors.extend(_check_tiie_hallucination(text, raw_data))
|
| 255 |
+
all_errors.extend(_check_math_operations(text))
|
| 256 |
+
|
| 257 |
+
if not all_errors:
|
| 258 |
+
return {"valid": True, "errors": [], "feedback": ""}
|
| 259 |
+
|
| 260 |
+
# Consolidar feedback
|
| 261 |
+
feedbacks = list({e["feedback"] for e in all_errors}) # deduplicar
|
| 262 |
+
feedback_msg = "\n".join(f"• {f}" for f in feedbacks)
|
| 263 |
+
|
| 264 |
+
# Registrar en catálogo
|
| 265 |
+
_log_errors(all_errors)
|
| 266 |
+
|
| 267 |
+
return {
|
| 268 |
+
"valid": False,
|
| 269 |
+
"errors": all_errors,
|
| 270 |
+
"feedback": feedback_msg,
|
| 271 |
+
"count": len(all_errors),
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def _log_errors(errors: list):
|
| 276 |
+
"""Registra errores detectados en el catálogo para análisis posterior."""
|
| 277 |
+
try:
|
| 278 |
+
catalog = load_error_catalog()
|
| 279 |
+
for err in errors:
|
| 280 |
+
# Actualizar contador de regla
|
| 281 |
+
for regla in catalog.get("reglas_escala", []):
|
| 282 |
+
if regla.get("id") == err.get("tipo"):
|
| 283 |
+
regla["detectado"] = regla.get("detectado", 0) + 1
|
| 284 |
+
|
| 285 |
+
# Agregar al log de errores detectados (máx 100)
|
| 286 |
+
catalog["errores_detectados"].append({
|
| 287 |
+
"timestamp": datetime.now().isoformat(),
|
| 288 |
+
"tipo": err.get("tipo"),
|
| 289 |
+
"texto": err.get("texto", "")[:100],
|
| 290 |
+
})
|
| 291 |
+
# Mantener solo los últimos 100
|
| 292 |
+
catalog["errores_detectados"] = catalog["errores_detectados"][-100:]
|
| 293 |
+
|
| 294 |
+
save_error_catalog(catalog)
|
| 295 |
+
except Exception:
|
| 296 |
+
pass
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
# ─── Loop completo: Generador → Evaluador → Corrector ────────────────────────
|
| 300 |
+
|
| 301 |
+
def critic_loop(
|
| 302 |
+
draft: str,
|
| 303 |
+
question: str,
|
| 304 |
+
raw_data: dict,
|
| 305 |
+
call_model_fn, # función call_model del orquestador
|
| 306 |
+
model: str,
|
| 307 |
+
system_prompt: str,
|
| 308 |
+
) -> tuple:
|
| 309 |
+
"""
|
| 310 |
+
Ejecuta el loop Generador → Evaluador → Corrector.
|
| 311 |
+
|
| 312 |
+
Args:
|
| 313 |
+
draft: Respuesta borrador del LLM
|
| 314 |
+
question: Pregunta original del usuario
|
| 315 |
+
raw_data: Datos crudos del bucket
|
| 316 |
+
call_model_fn: función call_model(model, messages, max_tokens)
|
| 317 |
+
model: Modelo a usar para corrección (LLaMA 8B)
|
| 318 |
+
system_prompt: System prompt del orquestador
|
| 319 |
+
|
| 320 |
+
Returns:
|
| 321 |
+
(respuesta_final: str, iteraciones: int, errores_detectados: int)
|
| 322 |
+
"""
|
| 323 |
+
current_draft = draft
|
| 324 |
+
total_errors = 0
|
| 325 |
+
|
| 326 |
+
for iteration in range(MAX_ITER):
|
| 327 |
+
# Paso 1: Evaluador Python (determinístico, ~1ms)
|
| 328 |
+
result = evaluate_response(current_draft, raw_data)
|
| 329 |
+
|
| 330 |
+
if result["valid"]:
|
| 331 |
+
# Sin errores — respuesta lista
|
| 332 |
+
return current_draft, iteration, total_errors
|
| 333 |
+
|
| 334 |
+
# Paso 2: Hay errores — LLM reescribe con feedback específico
|
| 335 |
+
total_errors += result["count"]
|
| 336 |
+
|
| 337 |
+
correction_prompt = (
|
| 338 |
+
f"Reescribe la siguiente respuesta corrigiendo SOLO los errores indicados. "
|
| 339 |
+
f"NO cambies los datos numéricos correctos. NO agregues información nueva.\n\n"
|
| 340 |
+
f"ERRORES A CORREGIR:\n{result['feedback']}\n\n"
|
| 341 |
+
f"RESPUESTA ORIGINAL:\n{current_draft}\n\n"
|
| 342 |
+
f"PREGUNTA ORIGINAL: {question}\n\n"
|
| 343 |
+
f"Devuelve SOLO la respuesta corregida, sin explicaciones."
|
| 344 |
+
)
|
| 345 |
+
|
| 346 |
+
msgs = [
|
| 347 |
+
{"role": "system", "content": system_prompt},
|
| 348 |
+
{"role": "user", "content": correction_prompt}
|
| 349 |
+
]
|
| 350 |
+
|
| 351 |
+
corrected, error = call_model_fn(model, msgs, max_tokens=800)
|
| 352 |
+
|
| 353 |
+
if error or not corrected:
|
| 354 |
+
# Si falla la corrección, devolver el borrador original
|
| 355 |
+
return current_draft, iteration + 1, total_errors
|
| 356 |
+
|
| 357 |
+
current_draft = corrected
|
| 358 |
+
|
| 359 |
+
# Máximo de iteraciones alcanzado — devolver última versión
|
| 360 |
+
return current_draft, MAX_ITER, total_errors
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
# ─── Helper para cargar datos del bucket ─────────────────────────────────────
|
| 364 |
+
|
| 365 |
+
def load_raw_data_for_validation() -> dict:
|
| 366 |
+
"""
|
| 367 |
+
Carga datos crudos del bucket para validación cruzada.
|
| 368 |
+
Solo carga lo necesario — no el histórico completo.
|
| 369 |
+
"""
|
| 370 |
+
result = {}
|
| 371 |
+
files = {
|
| 372 |
+
"banxico": os.path.join(DATA_PATH, "banxico.json"),
|
| 373 |
+
"worldbank": os.path.join(DATA_PATH, "worldbank.json"),
|
| 374 |
+
"ift": os.path.join(DATA_PATH, "ift.json"),
|
| 375 |
+
}
|
| 376 |
+
for key, path in files.items():
|
| 377 |
+
try:
|
| 378 |
+
if os.path.exists(path):
|
| 379 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 380 |
+
result[key] = json.load(f)
|
| 381 |
+
except Exception:
|
| 382 |
+
pass
|
| 383 |
+
return result
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
# ─── Estado del catálogo de errores ──────────────────────────────────────────
|
| 387 |
+
|
| 388 |
+
def get_error_stats() -> dict:
|
| 389 |
+
"""Retorna estadísticas del catálogo para la UI de administración."""
|
| 390 |
+
catalog = load_error_catalog()
|
| 391 |
+
total = len(catalog.get("errores_detectados", []))
|
| 392 |
+
por_tipo = {}
|
| 393 |
+
for err in catalog.get("errores_detectados", []):
|
| 394 |
+
t = err.get("tipo", "UNKNOWN")
|
| 395 |
+
por_tipo[t] = por_tipo.get(t, 0) + 1
|
| 396 |
+
return {
|
| 397 |
+
"total_errores_detectados": total,
|
| 398 |
+
"por_tipo": por_tipo,
|
| 399 |
+
"reglas_activas": len(catalog.get("reglas_escala", [])),
|
| 400 |
+
"ultimo_error": catalog["errores_detectados"][-1] if catalog.get("errores_detectados") else None,
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
if __name__ == "__main__":
|
| 405 |
+
# Test del evaluador
|
| 406 |
+
print("🧪 Test Critic-Loop\n")
|
| 407 |
+
|
| 408 |
+
test_cases = [
|
| 409 |
+
{
|
| 410 |
+
"texto": "Las exportaciones de México en 2024 fueron de 681.35 billones USD según el World Bank.",
|
| 411 |
+
"esperado": "ERROR SCALE_001 detectado"
|
| 412 |
+
},
|
| 413 |
+
{
|
| 414 |
+
"texto": "La inflación en México fue del 127.5% según el INPC.",
|
| 415 |
+
"esperado": "ERROR SCALE_002 detectado"
|
| 416 |
+
},
|
| 417 |
+
{
|
| 418 |
+
"texto": "Las exportaciones de México fueron $681 mil millones USD. La tasa objetivo es 6.5%.",
|
| 419 |
+
"esperado": "Sin errores"
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"texto": "Telcel generó 59,113 millones MXN, lo que representa el 39.8% de 148,541 millones.",
|
| 423 |
+
"esperado": "Sin errores (59113/148541 = 39.8% ✓)"
|
| 424 |
+
},
|
| 425 |
+
{
|
| 426 |
+
"texto": "Telcel generó 59,113 millones MXN, lo que representa el 55% de 148,541 millones.",
|
| 427 |
+
"esperado": "ERROR MATH_001 detectado (55% ≠ 39.8%)"
|
| 428 |
+
},
|
| 429 |
+
]
|
| 430 |
+
|
| 431 |
+
for i, case in enumerate(test_cases, 1):
|
| 432 |
+
result = evaluate_response(case["texto"], {})
|
| 433 |
+
status = "✅ VÁLIDO" if result["valid"] else f"❌ {len(result['errors'])} ERROR(ES)"
|
| 434 |
+
print(f"Test {i}: {status}")
|
| 435 |
+
print(f" Texto: {case['texto'][:80]}...")
|
| 436 |
+
print(f" Esperado: {case['esperado']}")
|
| 437 |
+
if not result["valid"]:
|
| 438 |
+
for err in result["errors"]:
|
| 439 |
+
print(f" → [{err['tipo']}] {err['feedback'][:100]}")
|
| 440 |
+
print()
|
modules/observability.py
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · Observabilidad — Métricas del sistema
|
| 3 |
+
===============================================
|
| 4 |
+
Registra latencia, tokens, costo estimado y rutas del pipeline.
|
| 5 |
+
|
| 6 |
+
Métricas por query:
|
| 7 |
+
- Latencia total y por etapa (clasificador, tools, síntesis, validador)
|
| 8 |
+
- Tokens input/output por modelo
|
| 9 |
+
- Costo estimado USD (basado en pricing NVIDIA NIM)
|
| 10 |
+
- Ruta tomada (SIMPLE/NUMÉRICA/NARRATIVA)
|
| 11 |
+
- Tool calls ejecutadas
|
| 12 |
+
- Cache hit/miss
|
| 13 |
+
- Errores del Critic-Loop detectados
|
| 14 |
+
|
| 15 |
+
Persistencia: /data/metrics.jsonl (append-only, máx 1000 líneas)
|
| 16 |
+
UI: tabla en Administración + resumen de sesión
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
import os
|
| 20 |
+
import json
|
| 21 |
+
import time
|
| 22 |
+
import threading
|
| 23 |
+
from datetime import datetime
|
| 24 |
+
from collections import defaultdict
|
| 25 |
+
|
| 26 |
+
METRICS_FILE = "/data/metrics.jsonl"
|
| 27 |
+
MAX_LINES = 1000
|
| 28 |
+
|
| 29 |
+
# Pricing NVIDIA NIM (USD por 1M tokens — estimado)
|
| 30 |
+
PRICING = {
|
| 31 |
+
"meta/llama-3.1-8b-instruct": {"input": 0.20, "output": 0.20},
|
| 32 |
+
"meta/llama-3.1-70b-instruct": {"input": 0.99, "output": 0.99},
|
| 33 |
+
"mistralai/mixtral-8x7b-instruct-v0.1": {"input": 0.60, "output": 0.60},
|
| 34 |
+
"nvidia/nv-embedqa-e5-v5": {"input": 0.06, "output": 0.0},
|
| 35 |
+
"nvidia/nv-rerankqa-mistral-4b-v3": {"input": 0.08, "output": 0.0},
|
| 36 |
+
"default": {"input": 0.50, "output": 0.50},
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
_lock = threading.Lock()
|
| 40 |
+
_session_metrics = [] # métricas de la sesión actual en RAM
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
# ─── Contexto de medición ─────────────────────────────────────────────────────
|
| 44 |
+
|
| 45 |
+
class QueryTrace:
|
| 46 |
+
"""
|
| 47 |
+
Contexto de trazabilidad para una query completa.
|
| 48 |
+
Uso:
|
| 49 |
+
trace = QueryTrace(question)
|
| 50 |
+
trace.start_stage("clasificador")
|
| 51 |
+
... código ...
|
| 52 |
+
trace.end_stage("clasificador")
|
| 53 |
+
trace.record_tokens("llama-70b", input=150, output=300)
|
| 54 |
+
trace.set_route("NARRATIVA")
|
| 55 |
+
trace.add_tool("read_banxico")
|
| 56 |
+
metrics = trace.finish()
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
def __init__(self, question: str):
|
| 60 |
+
self.question = question[:120]
|
| 61 |
+
self.start_time = time.time()
|
| 62 |
+
self.stages = {}
|
| 63 |
+
self.tokens = defaultdict(lambda: {"input": 0, "output": 0})
|
| 64 |
+
self.route = "UNKNOWN"
|
| 65 |
+
self.tools_used = []
|
| 66 |
+
self.cache_hit = False
|
| 67 |
+
self.critic_errors = 0
|
| 68 |
+
self.critic_iters = 0
|
| 69 |
+
self._stage_start = {}
|
| 70 |
+
|
| 71 |
+
def start_stage(self, stage: str):
|
| 72 |
+
self._stage_start[stage] = time.time()
|
| 73 |
+
|
| 74 |
+
def end_stage(self, stage: str):
|
| 75 |
+
if stage in self._stage_start:
|
| 76 |
+
ms = int((time.time() - self._stage_start[stage]) * 1000)
|
| 77 |
+
self.stages[stage] = ms
|
| 78 |
+
|
| 79 |
+
def record_tokens(self, model: str, input_tokens: int = 0, output_tokens: int = 0):
|
| 80 |
+
self.tokens[model]["input"] += input_tokens
|
| 81 |
+
self.tokens[model]["output"] += output_tokens
|
| 82 |
+
|
| 83 |
+
def set_route(self, route: str):
|
| 84 |
+
self.route = route
|
| 85 |
+
|
| 86 |
+
def add_tool(self, tool_name: str):
|
| 87 |
+
if tool_name not in self.tools_used:
|
| 88 |
+
self.tools_used.append(tool_name)
|
| 89 |
+
|
| 90 |
+
def set_cache_hit(self, hit: bool):
|
| 91 |
+
self.cache_hit = hit
|
| 92 |
+
|
| 93 |
+
def set_critic(self, errors: int, iterations: int):
|
| 94 |
+
self.critic_errors = errors
|
| 95 |
+
self.critic_iters = iterations
|
| 96 |
+
|
| 97 |
+
def finish(self) -> dict:
|
| 98 |
+
total_ms = int((time.time() - self.start_time) * 1000)
|
| 99 |
+
|
| 100 |
+
# Calcular costo estimado
|
| 101 |
+
total_cost_usd = 0.0
|
| 102 |
+
token_summary = {}
|
| 103 |
+
for model, counts in self.tokens.items():
|
| 104 |
+
pricing = PRICING.get(model, PRICING["default"])
|
| 105 |
+
cost = (counts["input"] / 1_000_000 * pricing["input"] +
|
| 106 |
+
counts["output"] / 1_000_000 * pricing["output"])
|
| 107 |
+
total_cost_usd += cost
|
| 108 |
+
token_summary[model] = {
|
| 109 |
+
"input": counts["input"],
|
| 110 |
+
"output": counts["output"],
|
| 111 |
+
"cost_usd": round(cost, 6),
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
metric = {
|
| 115 |
+
"ts": datetime.now().isoformat(),
|
| 116 |
+
"question": self.question,
|
| 117 |
+
"route": self.route,
|
| 118 |
+
"total_ms": total_ms,
|
| 119 |
+
"stages_ms": self.stages,
|
| 120 |
+
"tokens": token_summary,
|
| 121 |
+
"cost_usd": round(total_cost_usd, 6),
|
| 122 |
+
"tools_used": self.tools_used,
|
| 123 |
+
"cache_hit": self.cache_hit,
|
| 124 |
+
"critic_errors": self.critic_errors,
|
| 125 |
+
"critic_iters": self.critic_iters,
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
record_metric(metric)
|
| 129 |
+
return metric
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# ─── Persistencia ─────────────────────────────────────────────────────────────
|
| 133 |
+
|
| 134 |
+
def record_metric(metric: dict):
|
| 135 |
+
"""Guarda métrica en /data/metrics.jsonl y en RAM."""
|
| 136 |
+
global _session_metrics
|
| 137 |
+
with _lock:
|
| 138 |
+
_session_metrics.append(metric)
|
| 139 |
+
_session_metrics = _session_metrics[-500:] # máx 500 en RAM
|
| 140 |
+
|
| 141 |
+
try:
|
| 142 |
+
os.makedirs("/data", exist_ok=True)
|
| 143 |
+
with open(METRICS_FILE, "a", encoding="utf-8") as f:
|
| 144 |
+
f.write(json.dumps(metric, ensure_ascii=False) + "\n")
|
| 145 |
+
_trim_metrics_file()
|
| 146 |
+
except Exception:
|
| 147 |
+
pass
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _trim_metrics_file():
|
| 151 |
+
"""Mantiene el archivo de métricas en máximo MAX_LINES líneas."""
|
| 152 |
+
try:
|
| 153 |
+
if not os.path.exists(METRICS_FILE):
|
| 154 |
+
return
|
| 155 |
+
with open(METRICS_FILE, "r") as f:
|
| 156 |
+
lines = f.readlines()
|
| 157 |
+
if len(lines) > MAX_LINES:
|
| 158 |
+
with open(METRICS_FILE, "w") as f:
|
| 159 |
+
f.writelines(lines[-MAX_LINES:])
|
| 160 |
+
except Exception:
|
| 161 |
+
pass
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def load_metrics(last_n: int = 100) -> list:
|
| 165 |
+
"""Carga las últimas N métricas del archivo."""
|
| 166 |
+
try:
|
| 167 |
+
if not os.path.exists(METRICS_FILE):
|
| 168 |
+
return []
|
| 169 |
+
with open(METRICS_FILE, "r", encoding="utf-8") as f:
|
| 170 |
+
lines = f.readlines()
|
| 171 |
+
metrics = []
|
| 172 |
+
for line in lines[-last_n:]:
|
| 173 |
+
try:
|
| 174 |
+
metrics.append(json.loads(line.strip()))
|
| 175 |
+
except Exception:
|
| 176 |
+
pass
|
| 177 |
+
return metrics
|
| 178 |
+
except Exception:
|
| 179 |
+
return []
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
# ─── Análisis y estadísticas ──────────────────────────────────────────────────
|
| 183 |
+
|
| 184 |
+
def get_session_stats() -> dict:
|
| 185 |
+
"""Estadísticas de la sesión actual (desde el último arranque)."""
|
| 186 |
+
if not _session_metrics:
|
| 187 |
+
return {"queries": 0, "message": "Sin métricas en esta sesión"}
|
| 188 |
+
|
| 189 |
+
queries = len(_session_metrics)
|
| 190 |
+
total_ms = sum(m.get("total_ms", 0) for m in _session_metrics)
|
| 191 |
+
cache_hits = sum(1 for m in _session_metrics if m.get("cache_hit"))
|
| 192 |
+
total_cost = sum(m.get("cost_usd", 0) for m in _session_metrics)
|
| 193 |
+
critic_total = sum(m.get("critic_errors", 0) for m in _session_metrics)
|
| 194 |
+
|
| 195 |
+
# Latencia por ruta
|
| 196 |
+
by_route = defaultdict(list)
|
| 197 |
+
for m in _session_metrics:
|
| 198 |
+
by_route[m.get("route", "UNKNOWN")].append(m.get("total_ms", 0))
|
| 199 |
+
|
| 200 |
+
route_stats = {
|
| 201 |
+
route: {
|
| 202 |
+
"count": len(times),
|
| 203 |
+
"avg_ms": int(sum(times) / len(times)),
|
| 204 |
+
"min_ms": min(times),
|
| 205 |
+
"max_ms": max(times),
|
| 206 |
+
}
|
| 207 |
+
for route, times in by_route.items()
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
# Tools más usadas
|
| 211 |
+
tool_counts = defaultdict(int)
|
| 212 |
+
for m in _session_metrics:
|
| 213 |
+
for t in m.get("tools_used", []):
|
| 214 |
+
tool_counts[t] += 1
|
| 215 |
+
|
| 216 |
+
return {
|
| 217 |
+
"queries": queries,
|
| 218 |
+
"avg_latencia_ms": int(total_ms / queries) if queries else 0,
|
| 219 |
+
"cache_hits": cache_hits,
|
| 220 |
+
"cache_hit_rate": f"{cache_hits/queries*100:.0f}%" if queries else "0%",
|
| 221 |
+
"costo_total_usd": round(total_cost, 4),
|
| 222 |
+
"costo_por_query": round(total_cost / queries, 6) if queries else 0,
|
| 223 |
+
"critic_errors": critic_total,
|
| 224 |
+
"por_ruta": route_stats,
|
| 225 |
+
"tools_mas_usadas": dict(sorted(tool_counts.items(),
|
| 226 |
+
key=lambda x: x[1], reverse=True)[:5]),
|
| 227 |
+
"ultima_query": _session_metrics[-1].get("question", "")[:60] if _session_metrics else "",
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def get_metrics_for_ui(last_n: int = 20) -> list:
|
| 232 |
+
"""
|
| 233 |
+
Retorna métricas formateadas para la tabla de administración.
|
| 234 |
+
"""
|
| 235 |
+
metrics = load_metrics(last_n)
|
| 236 |
+
rows = []
|
| 237 |
+
for m in reversed(metrics):
|
| 238 |
+
ts = m.get("ts", "")[:16].replace("T", " ")
|
| 239 |
+
q = m.get("question", "")[:50]
|
| 240 |
+
route = m.get("route", "?")
|
| 241 |
+
ms = m.get("total_ms", 0)
|
| 242 |
+
cost = m.get("cost_usd", 0)
|
| 243 |
+
cache = "✅ HIT" if m.get("cache_hit") else "❌ MISS"
|
| 244 |
+
tools = ", ".join(m.get("tools_used", []))
|
| 245 |
+
rows.append({
|
| 246 |
+
"ts": ts,
|
| 247 |
+
"query": q,
|
| 248 |
+
"route": route,
|
| 249 |
+
"ms": ms,
|
| 250 |
+
"cost": f"${cost:.5f}",
|
| 251 |
+
"cache": cache,
|
| 252 |
+
"tools": tools,
|
| 253 |
+
})
|
| 254 |
+
return rows
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
# ─── Decorador para medir funciones ──────────────────────────────────────────
|
| 258 |
+
|
| 259 |
+
def measure(stage_name: str, trace: QueryTrace = None):
|
| 260 |
+
"""
|
| 261 |
+
Decorador/context manager para medir latencia de una etapa.
|
| 262 |
+
Uso: with measure("clasificador", trace): ...
|
| 263 |
+
"""
|
| 264 |
+
class _Ctx:
|
| 265 |
+
def __enter__(self):
|
| 266 |
+
if trace:
|
| 267 |
+
trace.start_stage(stage_name)
|
| 268 |
+
self._t = time.time()
|
| 269 |
+
return self
|
| 270 |
+
|
| 271 |
+
def __exit__(self, *args):
|
| 272 |
+
if trace:
|
| 273 |
+
trace.end_stage(stage_name)
|
| 274 |
+
|
| 275 |
+
return _Ctx()
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
# ─── Integración con call_model ───────────────────────────────────────────────
|
| 279 |
+
|
| 280 |
+
def wrap_call_model(original_call_model, trace: QueryTrace = None):
|
| 281 |
+
"""
|
| 282 |
+
Wrapper para call_model que registra tokens automáticamente.
|
| 283 |
+
Uso en agent_orchestrator.py:
|
| 284 |
+
result = wrap_call_model(call_model, trace)(model, messages, max_tokens)
|
| 285 |
+
"""
|
| 286 |
+
def wrapped(model: str, messages: list, max_tokens: int = 700):
|
| 287 |
+
t0 = time.time()
|
| 288 |
+
result, error = original_call_model(model, messages, max_tokens)
|
| 289 |
+
ms = int((time.time() - t0) * 1000)
|
| 290 |
+
|
| 291 |
+
if trace and result:
|
| 292 |
+
# Estimación de tokens (4 chars ≈ 1 token)
|
| 293 |
+
input_tokens = sum(len(m.get("content", "")) for m in messages) // 4
|
| 294 |
+
output_tokens = len(result) // 4
|
| 295 |
+
trace.record_tokens(model, input_tokens, output_tokens)
|
| 296 |
+
stage = model.split("/")[-1][:20]
|
| 297 |
+
trace.stages[stage] = ms
|
| 298 |
+
|
| 299 |
+
return result, error
|
| 300 |
+
return wrapped
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
if __name__ == "__main__":
|
| 304 |
+
print("🧪 Test Observabilidad\n")
|
| 305 |
+
|
| 306 |
+
# Simular una query completa
|
| 307 |
+
trace = QueryTrace("¿Cuál es la tasa objetivo del Banxico hoy?")
|
| 308 |
+
trace.start_stage("clasificador")
|
| 309 |
+
time.sleep(0.05)
|
| 310 |
+
trace.end_stage("clasificador")
|
| 311 |
+
trace.set_route("SIMPLE")
|
| 312 |
+
trace.add_tool("read_banxico")
|
| 313 |
+
trace.record_tokens("meta/llama-3.1-8b-instruct", 50, 10)
|
| 314 |
+
trace.record_tokens("meta/llama-3.1-70b-instruct", 200, 150)
|
| 315 |
+
trace.set_cache_hit(False)
|
| 316 |
+
trace.set_critic(errors=0, iterations=1)
|
| 317 |
+
|
| 318 |
+
metric = trace.finish()
|
| 319 |
+
print(f"✅ Query registrada:")
|
| 320 |
+
print(f" Ruta: {metric['route']}")
|
| 321 |
+
print(f" Latencia total: {metric['total_ms']}ms")
|
| 322 |
+
print(f" Costo estimado: ${metric['cost_usd']:.6f} USD")
|
| 323 |
+
print(f" Tools: {metric['tools_used']}")
|
| 324 |
+
print(f" Stages: {metric['stages_ms']}")
|
| 325 |
+
|
| 326 |
+
stats = get_session_stats()
|
| 327 |
+
print(f"\n📊 Stats sesión:")
|
| 328 |
+
print(f" Queries: {stats['queries']}")
|
| 329 |
+
print(f" Avg latencia: {stats['avg_latencia_ms']}ms")
|
| 330 |
+
print(f" Costo total: ${stats['costo_total_usd']} USD")
|
modules/rag_chroma.py
ADDED
|
@@ -0,0 +1,828 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · RAG con ChromaDB
|
| 3 |
+
==========================
|
| 4 |
+
Embeddings via NVIDIA NIM API (nvidia/nv-embedqa-e5-v5) — H100.
|
| 5 |
+
NO usa SentenceTransformer local — todo por NVIDIA API.
|
| 6 |
+
|
| 7 |
+
Colecciones:
|
| 8 |
+
- banxico_historico → series históricas 2024-2026
|
| 9 |
+
- banxico_actual → indicadores actuales Banxico
|
| 10 |
+
- worldbank → 22 indicadores World Bank
|
| 11 |
+
- inegi → indicadores INEGI
|
| 12 |
+
- ift → sector telecomunicaciones México
|
| 13 |
+
|
| 14 |
+
Flujo:
|
| 15 |
+
1. Al arrancar: indexar JSONs del bucket → ChromaDB en /data/chroma_db/
|
| 16 |
+
2. Embeddings via NVIDIA NIM — misma H100 que los LLMs
|
| 17 |
+
3. El orquestador llama search_chroma() para búsqueda semántica
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import hashlib
|
| 23 |
+
import requests
|
| 24 |
+
from datetime import datetime
|
| 25 |
+
|
| 26 |
+
DATA_PATH = "/data"
|
| 27 |
+
CHROMA_PATH = "/tmp/chroma_db"
|
| 28 |
+
MODEL_NAME = "nvidia/nv-embedqa-e5-v5"
|
| 29 |
+
NVIDIA_BASE = "https://integrate.api.nvidia.com/v1"
|
| 30 |
+
EMBED_DIM = 1024 # dimensión de nv-embedqa-e5-v5
|
| 31 |
+
|
| 32 |
+
_chroma_client = None
|
| 33 |
+
_collections = {}
|
| 34 |
+
_index_status = {}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _get_nvidia_key() -> str:
|
| 38 |
+
return os.getenv("NVIDIA_API_KEY", "")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ─── Embeddings via NVIDIA NIM ────────────────────────────────────────────────
|
| 42 |
+
|
| 43 |
+
# ─── Embedding strategy ──────────────────────────────────────────────────────
|
| 44 |
+
# INDEXING: SentenceTransformer local (fast, ~30s for 22k docs, no API calls)
|
| 45 |
+
# QUERYING: NVIDIA NIM (high quality semantic search, 1 call per query)
|
| 46 |
+
|
| 47 |
+
_st_model = None
|
| 48 |
+
|
| 49 |
+
def _get_local_model():
|
| 50 |
+
"""Lazy load SentenceTransformer for indexing."""
|
| 51 |
+
global _st_model
|
| 52 |
+
if _st_model is None:
|
| 53 |
+
from sentence_transformers import SentenceTransformer
|
| 54 |
+
_st_model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
|
| 55 |
+
return _st_model
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _get_embeddings(texts: list, input_type: str = "passage") -> list:
|
| 59 |
+
"""
|
| 60 |
+
Genera embeddings:
|
| 61 |
+
- "passage" (indexado) → SentenceTransformer local (384d, rápido)
|
| 62 |
+
- "query" (búsqueda) → NVIDIA NIM si disponible, sino local
|
| 63 |
+
"""
|
| 64 |
+
if input_type == "passage":
|
| 65 |
+
# Indexado: siempre local para velocidad
|
| 66 |
+
model = _get_local_model()
|
| 67 |
+
return model.encode(texts, batch_size=128, show_progress_bar=False).tolist()
|
| 68 |
+
|
| 69 |
+
# Búsqueda: NVIDIA NIM si hay API key
|
| 70 |
+
api_key = _get_nvidia_key()
|
| 71 |
+
if api_key:
|
| 72 |
+
try:
|
| 73 |
+
headers = {
|
| 74 |
+
"Authorization": f"Bearer {api_key}",
|
| 75 |
+
"Content-Type": "application/json",
|
| 76 |
+
}
|
| 77 |
+
r = requests.post(
|
| 78 |
+
f"{NVIDIA_BASE}/embeddings",
|
| 79 |
+
headers=headers,
|
| 80 |
+
json={
|
| 81 |
+
"model": MODEL_NAME,
|
| 82 |
+
"input": texts,
|
| 83 |
+
"input_type": "query",
|
| 84 |
+
"encoding_format": "float",
|
| 85 |
+
"truncate": "END",
|
| 86 |
+
},
|
| 87 |
+
timeout=15,
|
| 88 |
+
)
|
| 89 |
+
r.raise_for_status()
|
| 90 |
+
data = sorted(r.json()["data"], key=lambda x: x["index"])
|
| 91 |
+
return [item["embedding"] for item in data]
|
| 92 |
+
except Exception:
|
| 93 |
+
pass
|
| 94 |
+
|
| 95 |
+
# Fallback: local para búsqueda también
|
| 96 |
+
model = _get_local_model()
|
| 97 |
+
return model.encode(texts, show_progress_bar=False).tolist()
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# ─── ChromaDB client ──────────────────────────────────────────────────────────
|
| 101 |
+
|
| 102 |
+
def _get_chroma_client():
|
| 103 |
+
global _chroma_client
|
| 104 |
+
if _chroma_client is None:
|
| 105 |
+
import chromadb
|
| 106 |
+
os.makedirs(CHROMA_PATH, exist_ok=True)
|
| 107 |
+
_chroma_client = chromadb.EphemeralClient()
|
| 108 |
+
return _chroma_client
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _get_collection(name: str):
|
| 112 |
+
if name not in _collections:
|
| 113 |
+
client = _get_chroma_client()
|
| 114 |
+
_collections[name] = client.get_or_create_collection(
|
| 115 |
+
name=name,
|
| 116 |
+
metadata={"hnsw:space": "cosine"}
|
| 117 |
+
)
|
| 118 |
+
return _collections[name]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _file_hash(path: str) -> str:
|
| 122 |
+
try:
|
| 123 |
+
with open(path, "rb") as f:
|
| 124 |
+
return hashlib.md5(f.read()).hexdigest()
|
| 125 |
+
except Exception:
|
| 126 |
+
return ""
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _needs_reindex(collection_name: str, file_path: str) -> bool:
|
| 130 |
+
current_hash = _file_hash(file_path)
|
| 131 |
+
last_hash = _index_status.get(collection_name, {}).get("hash", "")
|
| 132 |
+
return current_hash != last_hash
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
# ─── Indexadores por fuente ───────────────────────────────────────────────────
|
| 136 |
+
|
| 137 |
+
def index_banxico_historico(force: bool = False) -> dict:
|
| 138 |
+
path = os.path.join(DATA_PATH, "banxico_historico.json")
|
| 139 |
+
collection_name = "banxico_historico"
|
| 140 |
+
|
| 141 |
+
if not force and not _needs_reindex(collection_name, path):
|
| 142 |
+
return {"status": "skip", "reason": "Sin cambios desde último indexado"}
|
| 143 |
+
|
| 144 |
+
try:
|
| 145 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 146 |
+
data = json.load(f)
|
| 147 |
+
|
| 148 |
+
collection = _get_collection(collection_name)
|
| 149 |
+
nombres = {
|
| 150 |
+
"SF43718": "Tipo de cambio USD/MXN",
|
| 151 |
+
"SF43783": "TIIE 28 días",
|
| 152 |
+
"SF61745": "Tasa objetivo Banxico",
|
| 153 |
+
"SP1": "Inflación IPC",
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
documents, metadatas, ids = [], [], []
|
| 157 |
+
|
| 158 |
+
for serie_id, registros in data.items():
|
| 159 |
+
if serie_id in ["fuente", "timestamp", "periodo"] or not isinstance(registros, list):
|
| 160 |
+
continue
|
| 161 |
+
nombre_serie = nombres.get(serie_id, serie_id)
|
| 162 |
+
# Limitar a últimos 3000 registros para velocidad de indexado
|
| 163 |
+
# Los datos más antiguos se sirven directo del JSON via _search_historico_by_year
|
| 164 |
+
registros = registros[-3000:] if len(registros) > 3000 else registros
|
| 165 |
+
for i, registro in enumerate(registros):
|
| 166 |
+
fecha = registro.get("fecha", registro.get("date", "sin_fecha"))
|
| 167 |
+
valor = registro.get("dato", registro.get("valor", registro.get("value", "N/A")))
|
| 168 |
+
texto = (
|
| 169 |
+
f"{nombre_serie} ({serie_id}) el {fecha}: {valor}. "
|
| 170 |
+
f"Serie: {serie_id}. "
|
| 171 |
+
f"Indicador Banxico de tipo {nombre_serie.lower()}."
|
| 172 |
+
)
|
| 173 |
+
documents.append(texto)
|
| 174 |
+
metadatas.append({
|
| 175 |
+
"serie": serie_id,
|
| 176 |
+
"nombre": nombre_serie,
|
| 177 |
+
"fecha": str(fecha),
|
| 178 |
+
"valor": str(valor),
|
| 179 |
+
"fuente": "Banxico SIE API",
|
| 180 |
+
})
|
| 181 |
+
ids.append(f"{serie_id}_{i}")
|
| 182 |
+
|
| 183 |
+
if not documents:
|
| 184 |
+
return {"status": "error", "reason": "Sin registros en banxico_historico.json"}
|
| 185 |
+
|
| 186 |
+
# Batch size 500 — SentenceTransformer local maneja bien lotes grandes
|
| 187 |
+
batch_size = 500
|
| 188 |
+
total = 0
|
| 189 |
+
model = _get_local_model()
|
| 190 |
+
for i in range(0, len(documents), batch_size):
|
| 191 |
+
batch_docs = documents[i:i+batch_size]
|
| 192 |
+
batch_meta = metadatas[i:i+batch_size]
|
| 193 |
+
batch_ids = ids[i:i+batch_size]
|
| 194 |
+
# Encodar todo el batch de una vez — más eficiente
|
| 195 |
+
embeddings = model.encode(
|
| 196 |
+
batch_docs,
|
| 197 |
+
batch_size=128,
|
| 198 |
+
show_progress_bar=False,
|
| 199 |
+
convert_to_numpy=True
|
| 200 |
+
).tolist()
|
| 201 |
+
collection.upsert(
|
| 202 |
+
documents=batch_docs,
|
| 203 |
+
embeddings=embeddings,
|
| 204 |
+
metadatas=batch_meta,
|
| 205 |
+
ids=batch_ids,
|
| 206 |
+
)
|
| 207 |
+
total += len(batch_docs)
|
| 208 |
+
|
| 209 |
+
_index_status[collection_name] = {
|
| 210 |
+
"indexed": True, "docs": total,
|
| 211 |
+
"hash": _file_hash(path), "ts": datetime.now().isoformat()
|
| 212 |
+
}
|
| 213 |
+
return {"status": "ok", "docs_indexados": total, "coleccion": collection_name}
|
| 214 |
+
|
| 215 |
+
except FileNotFoundError:
|
| 216 |
+
return {"status": "error", "reason": "banxico_historico.json no encontrado"}
|
| 217 |
+
except Exception as e:
|
| 218 |
+
return {"status": "error", "reason": str(e)}
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def index_banxico_actual(force: bool = False) -> dict:
|
| 222 |
+
path = os.path.join(DATA_PATH, "banxico.json")
|
| 223 |
+
collection_name = "banxico_actual"
|
| 224 |
+
|
| 225 |
+
if not force and not _needs_reindex(collection_name, path):
|
| 226 |
+
return {"status": "skip", "reason": "Sin cambios"}
|
| 227 |
+
|
| 228 |
+
try:
|
| 229 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 230 |
+
data = json.load(f)
|
| 231 |
+
|
| 232 |
+
collection = _get_collection(collection_name)
|
| 233 |
+
ts = data.get("timestamp", data.get("actualizado", "N/A"))
|
| 234 |
+
|
| 235 |
+
def extraer_indicadores(d, prefix=""):
|
| 236 |
+
items = []
|
| 237 |
+
for k, v in d.items():
|
| 238 |
+
if k in ["fuente", "timestamp", "actualizado", "archivo"]:
|
| 239 |
+
continue
|
| 240 |
+
if isinstance(v, dict):
|
| 241 |
+
valor = v.get("valor", v.get("valor_formateado", str(v)))
|
| 242 |
+
fecha = v.get("fecha", ts)
|
| 243 |
+
items.append((f"{prefix}{k}", str(valor), str(fecha)))
|
| 244 |
+
items.extend(extraer_indicadores(v, prefix=f"{k}."))
|
| 245 |
+
elif not isinstance(v, list):
|
| 246 |
+
items.append((f"{prefix}{k}", str(v), ts))
|
| 247 |
+
return items
|
| 248 |
+
|
| 249 |
+
indicadores = extraer_indicadores(data)
|
| 250 |
+
if not indicadores:
|
| 251 |
+
return {"status": "error", "reason": "Sin indicadores en banxico.json"}
|
| 252 |
+
|
| 253 |
+
documents, metadatas, ids = [], [], []
|
| 254 |
+
for nombre, valor, fecha in indicadores:
|
| 255 |
+
texto = (
|
| 256 |
+
f"Indicador Banxico {nombre}: {valor} (fecha: {fecha}). "
|
| 257 |
+
f"Dato actual del Banco de México."
|
| 258 |
+
)
|
| 259 |
+
documents.append(texto)
|
| 260 |
+
metadatas.append({"indicador": nombre, "valor": valor, "fecha": fecha, "fuente": "Banxico"})
|
| 261 |
+
ids.append(f"banxico_{nombre.replace('.','_')}")
|
| 262 |
+
|
| 263 |
+
embeddings = _get_embeddings(documents, input_type="passage")
|
| 264 |
+
collection.upsert(documents=documents, embeddings=embeddings,
|
| 265 |
+
metadatas=metadatas, ids=ids)
|
| 266 |
+
|
| 267 |
+
_index_status[collection_name] = {
|
| 268 |
+
"indexed": True, "docs": len(documents),
|
| 269 |
+
"hash": _file_hash(path), "ts": datetime.now().isoformat()
|
| 270 |
+
}
|
| 271 |
+
return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name}
|
| 272 |
+
|
| 273 |
+
except FileNotFoundError:
|
| 274 |
+
return {"status": "error", "reason": "banxico.json no encontrado"}
|
| 275 |
+
except Exception as e:
|
| 276 |
+
return {"status": "error", "reason": str(e)}
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def index_worldbank(force: bool = False) -> dict:
|
| 280 |
+
path = os.path.join(DATA_PATH, "worldbank.json")
|
| 281 |
+
collection_name = "worldbank"
|
| 282 |
+
|
| 283 |
+
if not force and not _needs_reindex(collection_name, path):
|
| 284 |
+
return {"status": "skip", "reason": "Sin cambios"}
|
| 285 |
+
|
| 286 |
+
try:
|
| 287 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 288 |
+
data = json.load(f)
|
| 289 |
+
|
| 290 |
+
collection = _get_collection(collection_name)
|
| 291 |
+
documents, metadatas, ids = [], [], []
|
| 292 |
+
|
| 293 |
+
for categoria, indicadores in data.items():
|
| 294 |
+
if categoria in ["fuente", "timestamp", "pais"] or not isinstance(indicadores, dict):
|
| 295 |
+
continue
|
| 296 |
+
for nombre, info in indicadores.items():
|
| 297 |
+
if isinstance(info, dict):
|
| 298 |
+
valor = info.get("valor_formateado", info.get("valor", str(info)))
|
| 299 |
+
anio = info.get("anio", info.get("year", "N/A"))
|
| 300 |
+
unidad = info.get("unidad", "")
|
| 301 |
+
else:
|
| 302 |
+
valor = str(info)
|
| 303 |
+
anio = "N/A"
|
| 304 |
+
unidad = ""
|
| 305 |
+
texto = (
|
| 306 |
+
f"World Bank México — {nombre} ({categoria}): {valor} {unidad} "
|
| 307 |
+
f"(año: {anio}). Indicador del Banco Mundial para México."
|
| 308 |
+
)
|
| 309 |
+
doc_id = f"wb_{categoria}_{nombre.replace(' ','_')[:40]}"
|
| 310 |
+
documents.append(texto)
|
| 311 |
+
metadatas.append({
|
| 312 |
+
"categoria": categoria, "indicador": nombre,
|
| 313 |
+
"valor": str(valor), "anio": str(anio), "fuente": "World Bank"
|
| 314 |
+
})
|
| 315 |
+
ids.append(doc_id)
|
| 316 |
+
|
| 317 |
+
if not documents:
|
| 318 |
+
return {"status": "error", "reason": "Sin datos en worldbank.json"}
|
| 319 |
+
|
| 320 |
+
embeddings = _get_embeddings(documents, input_type="passage")
|
| 321 |
+
collection.upsert(documents=documents, embeddings=embeddings,
|
| 322 |
+
metadatas=metadatas, ids=ids)
|
| 323 |
+
|
| 324 |
+
_index_status[collection_name] = {
|
| 325 |
+
"indexed": True, "docs": len(documents),
|
| 326 |
+
"hash": _file_hash(path), "ts": datetime.now().isoformat()
|
| 327 |
+
}
|
| 328 |
+
return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name}
|
| 329 |
+
|
| 330 |
+
except FileNotFoundError:
|
| 331 |
+
return {"status": "error", "reason": "worldbank.json no encontrado"}
|
| 332 |
+
except Exception as e:
|
| 333 |
+
return {"status": "error", "reason": str(e)}
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def index_inegi(force: bool = False) -> dict:
|
| 337 |
+
path = os.path.join(DATA_PATH, "inegi.json")
|
| 338 |
+
collection_name = "inegi"
|
| 339 |
+
|
| 340 |
+
if not force and not _needs_reindex(collection_name, path):
|
| 341 |
+
return {"status": "skip", "reason": "Sin cambios"}
|
| 342 |
+
|
| 343 |
+
try:
|
| 344 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 345 |
+
data = json.load(f)
|
| 346 |
+
|
| 347 |
+
collection = _get_collection(collection_name)
|
| 348 |
+
ts = data.get("timestamp", "N/A")
|
| 349 |
+
documents, metadatas, ids = [], [], []
|
| 350 |
+
|
| 351 |
+
def flatten(d, prefix=""):
|
| 352 |
+
for k, v in d.items():
|
| 353 |
+
if k in ["fuente", "timestamp", "detalle"]:
|
| 354 |
+
continue
|
| 355 |
+
if isinstance(v, dict):
|
| 356 |
+
flatten(v, prefix=f"{k}.")
|
| 357 |
+
elif not isinstance(v, list):
|
| 358 |
+
texto = (
|
| 359 |
+
f"INEGI México — {prefix}{k}: {v} "
|
| 360 |
+
f"(fecha extracción: {ts}). "
|
| 361 |
+
f"Indicador del Instituto Nacional de Estadística."
|
| 362 |
+
)
|
| 363 |
+
doc_id = f"inegi_{(prefix+k).replace('.','_')[:50]}"
|
| 364 |
+
documents.append(texto)
|
| 365 |
+
metadatas.append({
|
| 366 |
+
"indicador": f"{prefix}{k}", "valor": str(v),
|
| 367 |
+
"fecha": ts, "fuente": "INEGI"
|
| 368 |
+
})
|
| 369 |
+
ids.append(doc_id)
|
| 370 |
+
|
| 371 |
+
flatten(data)
|
| 372 |
+
if "detalle" in data and isinstance(data["detalle"], dict):
|
| 373 |
+
flatten(data["detalle"], prefix="detalle.")
|
| 374 |
+
|
| 375 |
+
if not documents:
|
| 376 |
+
return {"status": "error", "reason": "Sin datos en inegi.json"}
|
| 377 |
+
|
| 378 |
+
embeddings = _get_embeddings(documents, input_type="passage")
|
| 379 |
+
collection.upsert(documents=documents, embeddings=embeddings,
|
| 380 |
+
metadatas=metadatas, ids=ids)
|
| 381 |
+
|
| 382 |
+
_index_status[collection_name] = {
|
| 383 |
+
"indexed": True, "docs": len(documents),
|
| 384 |
+
"hash": _file_hash(path), "ts": datetime.now().isoformat()
|
| 385 |
+
}
|
| 386 |
+
return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name}
|
| 387 |
+
|
| 388 |
+
except FileNotFoundError:
|
| 389 |
+
return {"status": "error", "reason": "inegi.json no encontrado"}
|
| 390 |
+
except Exception as e:
|
| 391 |
+
return {"status": "error", "reason": str(e)}
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
def index_ift(force: bool = False) -> dict:
|
| 395 |
+
path = os.path.join(DATA_PATH, "ift.json")
|
| 396 |
+
collection_name = "ift"
|
| 397 |
+
|
| 398 |
+
if not force and not _needs_reindex(collection_name, path):
|
| 399 |
+
return {"status": "skip", "reason": "Sin cambios"}
|
| 400 |
+
|
| 401 |
+
try:
|
| 402 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 403 |
+
data = json.load(f)
|
| 404 |
+
|
| 405 |
+
collection = _get_collection(collection_name)
|
| 406 |
+
documents, metadatas, ids = [], [], []
|
| 407 |
+
periodo = data.get("metadatos", {}).get("periodo", "Q1 2025")
|
| 408 |
+
|
| 409 |
+
for op in data.get("ingresos_q1_2025", {}).get("operadores", []):
|
| 410 |
+
marca = op.get("marca", "")
|
| 411 |
+
grupo = op.get("grupo", "")
|
| 412 |
+
ingreso = op.get("ingreso_millones_mxn", "")
|
| 413 |
+
pct = op.get("participacion_pct", "")
|
| 414 |
+
texto = (
|
| 415 |
+
f"IFT Telecomunicaciones México — {marca} ({grupo}): "
|
| 416 |
+
f"ingresos {ingreso} millones MXN, participación {pct}% del sector. "
|
| 417 |
+
f"Período: {periodo}. Fuente: IFT datos oportunos."
|
| 418 |
+
)
|
| 419 |
+
documents.append(texto)
|
| 420 |
+
metadatas.append({
|
| 421 |
+
"categoria": "ingresos", "operador": marca,
|
| 422 |
+
"valor": str(ingreso), "participacion": str(pct),
|
| 423 |
+
"fuente": "IFT", "periodo": periodo
|
| 424 |
+
})
|
| 425 |
+
ids.append(f"ift_ingreso_{marca.replace(' ','_')[:30]}")
|
| 426 |
+
|
| 427 |
+
for op in data.get("participacion_mercado_movil", {}).get("operadores", []):
|
| 428 |
+
operador = op.get("operador", "")
|
| 429 |
+
lineas = op.get("lineas_millones", "")
|
| 430 |
+
pct = op.get("participacion_pct", "")
|
| 431 |
+
texto = (
|
| 432 |
+
f"IFT México mercado móvil — {operador}: "
|
| 433 |
+
f"{lineas} millones de líneas, {pct}% participación de mercado. "
|
| 434 |
+
f"Período: Q3 2024. Fuente: IFT."
|
| 435 |
+
)
|
| 436 |
+
documents.append(texto)
|
| 437 |
+
metadatas.append({
|
| 438 |
+
"categoria": "mercado_movil", "operador": operador,
|
| 439 |
+
"valor": str(lineas), "participacion": str(pct),
|
| 440 |
+
"fuente": "IFT", "periodo": "Q3 2024"
|
| 441 |
+
})
|
| 442 |
+
ids.append(f"ift_movil_{operador.replace(' ','_')[:30]}")
|
| 443 |
+
|
| 444 |
+
conn = data.get("indicadores_conectividad", {})
|
| 445 |
+
for k, v in conn.items():
|
| 446 |
+
if k in ["fuente", "periodo"] or isinstance(v, dict):
|
| 447 |
+
continue
|
| 448 |
+
texto = (
|
| 449 |
+
f"IFT México conectividad — {k}: {v}. "
|
| 450 |
+
f"Indicador de telecomunicaciones México. Fuente: IFT/ENDUTIH 2024."
|
| 451 |
+
)
|
| 452 |
+
documents.append(texto)
|
| 453 |
+
metadatas.append({
|
| 454 |
+
"categoria": "conectividad", "indicador": k,
|
| 455 |
+
"valor": str(v), "fuente": "IFT", "periodo": "2024"
|
| 456 |
+
})
|
| 457 |
+
ids.append(f"ift_conn_{k[:40]}")
|
| 458 |
+
|
| 459 |
+
cobertura = data.get("cobertura_5g", {})
|
| 460 |
+
texto_5g = (
|
| 461 |
+
f"IFT México cobertura 5G — Estado: {cobertura.get('estado_2024', '')}. "
|
| 462 |
+
f"Operadores con 5G: {', '.join(cobertura.get('operadores_con_5g', []))}. "
|
| 463 |
+
f"Ciudades principales 5G: {', '.join(cobertura.get('ciudades_principales_5g', []))}. "
|
| 464 |
+
f"Penetración 5G estimada: {cobertura.get('penetracion_5g_estimada_pct', '')}%. "
|
| 465 |
+
f"Cobertura 4G LTE: {cobertura.get('cobertura_4g_lte_pct_poblacion', '')}% población. "
|
| 466 |
+
f"Fuente: IFT/GSMA Intelligence 2024."
|
| 467 |
+
)
|
| 468 |
+
documents.append(texto_5g)
|
| 469 |
+
metadatas.append({
|
| 470 |
+
"categoria": "cobertura_5g", "indicador": "5G_4G",
|
| 471 |
+
"valor": str(cobertura.get("penetracion_5g_estimada_pct", "")),
|
| 472 |
+
"fuente": "IFT/GSMA", "periodo": "2024"
|
| 473 |
+
})
|
| 474 |
+
ids.append("ift_cobertura_5g")
|
| 475 |
+
|
| 476 |
+
reg = data.get("contexto_regulatorio", {})
|
| 477 |
+
texto_reg = (
|
| 478 |
+
f"IFT regulación telecomunicaciones México — "
|
| 479 |
+
f"Agente Económico Preponderante (AEP): {reg.get('agente_economico_preponderante', '')}. "
|
| 480 |
+
f"Regulador: {reg.get('regulador', '')}. "
|
| 481 |
+
f"{reg.get('nota_2025', '')} "
|
| 482 |
+
f"{reg.get('sancion_telcel_2025', '')}"
|
| 483 |
+
)
|
| 484 |
+
documents.append(texto_reg)
|
| 485 |
+
metadatas.append({
|
| 486 |
+
"categoria": "regulacion", "indicador": "AEP",
|
| 487 |
+
"valor": "America Movil", "fuente": "IFT", "periodo": "2025"
|
| 488 |
+
})
|
| 489 |
+
ids.append("ift_regulacion")
|
| 490 |
+
|
| 491 |
+
if not documents:
|
| 492 |
+
return {"status": "error", "reason": "Sin datos en ift.json"}
|
| 493 |
+
|
| 494 |
+
embeddings = _get_embeddings(documents, input_type="passage")
|
| 495 |
+
collection.upsert(documents=documents, embeddings=embeddings,
|
| 496 |
+
metadatas=metadatas, ids=ids)
|
| 497 |
+
|
| 498 |
+
_index_status[collection_name] = {
|
| 499 |
+
"indexed": True, "docs": len(documents),
|
| 500 |
+
"hash": _file_hash(path), "ts": datetime.now().isoformat()
|
| 501 |
+
}
|
| 502 |
+
return {"status": "ok", "docs_indexados": len(documents), "coleccion": collection_name}
|
| 503 |
+
|
| 504 |
+
except FileNotFoundError:
|
| 505 |
+
return {"status": "error", "reason": "ift.json no encontrado — ejecutar data_extractor.py"}
|
| 506 |
+
except Exception as e:
|
| 507 |
+
return {"status": "error", "reason": str(e)}
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
# ─── Indexado completo ────────────────────────────────────────────────────────
|
| 511 |
+
|
| 512 |
+
def index_all(force: bool = False) -> dict:
|
| 513 |
+
results = {
|
| 514 |
+
"banxico_historico": index_banxico_historico(force),
|
| 515 |
+
"banxico_actual": index_banxico_actual(force),
|
| 516 |
+
"worldbank": index_worldbank(force),
|
| 517 |
+
"inegi": index_inegi(force),
|
| 518 |
+
"ift": index_ift(force),
|
| 519 |
+
}
|
| 520 |
+
ok = sum(1 for r in results.values() if r.get("status") in ["ok", "skip"])
|
| 521 |
+
error = sum(1 for r in results.values() if r.get("status") == "error")
|
| 522 |
+
return {
|
| 523 |
+
"status": "ok" if error == 0 else "partial",
|
| 524 |
+
"ok": ok,
|
| 525 |
+
"errores": error,
|
| 526 |
+
"detalle": results,
|
| 527 |
+
"ts": datetime.now().isoformat(),
|
| 528 |
+
}
|
| 529 |
+
|
| 530 |
+
|
| 531 |
+
# ─── Búsqueda semántica ───────────────────────────────────────────────────────
|
| 532 |
+
|
| 533 |
+
def search_chroma(query: str, collection_name: str, n_results: int = 5,
|
| 534 |
+
where: dict = None) -> list:
|
| 535 |
+
try:
|
| 536 |
+
collection = _get_collection(collection_name)
|
| 537 |
+
|
| 538 |
+
if collection.count() == 0:
|
| 539 |
+
return [{"error": f"Colección '{collection_name}' vacía — ejecutar index_all()"}]
|
| 540 |
+
|
| 541 |
+
query_embedding = _get_embeddings([query], input_type="query")
|
| 542 |
+
|
| 543 |
+
kwargs = {
|
| 544 |
+
"query_embeddings": query_embedding,
|
| 545 |
+
"n_results": min(n_results, collection.count()),
|
| 546 |
+
"include": ["documents", "metadatas", "distances"],
|
| 547 |
+
}
|
| 548 |
+
if where:
|
| 549 |
+
kwargs["where"] = where
|
| 550 |
+
|
| 551 |
+
results = collection.query(**kwargs)
|
| 552 |
+
output = []
|
| 553 |
+
for doc, meta, dist in zip(
|
| 554 |
+
results["documents"][0],
|
| 555 |
+
results["metadatas"][0],
|
| 556 |
+
results["distances"][0],
|
| 557 |
+
):
|
| 558 |
+
output.append({
|
| 559 |
+
"texto": doc,
|
| 560 |
+
"metadata": meta,
|
| 561 |
+
"distancia": round(dist, 4),
|
| 562 |
+
"relevancia": round(1 - dist, 4),
|
| 563 |
+
})
|
| 564 |
+
return output
|
| 565 |
+
|
| 566 |
+
except Exception as e:
|
| 567 |
+
return [{"error": str(e)}]
|
| 568 |
+
|
| 569 |
+
|
| 570 |
+
def search_historico_semantico(query: str, serie: str = None,
|
| 571 |
+
n_results: int = 50) -> dict:
|
| 572 |
+
import re as _re
|
| 573 |
+
years_mentioned = _re.findall(r'\b(202[0-9])\b', query)
|
| 574 |
+
|
| 575 |
+
if years_mentioned:
|
| 576 |
+
return _search_historico_by_year(query, years_mentioned, serie)
|
| 577 |
+
|
| 578 |
+
q_lower = query.lower()
|
| 579 |
+
if any(k in q_lower for k in ["evoluc", "tendencia", "trend", "durante", "todo", "año", "year",
|
| 580 |
+
"máximo", "mínimo", "maximum", "minimum", "compara", "compare"]):
|
| 581 |
+
n_results = max(n_results, 100)
|
| 582 |
+
|
| 583 |
+
try:
|
| 584 |
+
where = {"serie": serie} if serie else None
|
| 585 |
+
results = search_chroma(query, "banxico_historico", n_results=n_results, where=where)
|
| 586 |
+
if results and "error" in results[0]:
|
| 587 |
+
results = search_chroma(query, "banxico_historico", n_results=n_results, where=None)
|
| 588 |
+
except Exception:
|
| 589 |
+
results = search_chroma(query, "banxico_historico", n_results=n_results, where=None)
|
| 590 |
+
|
| 591 |
+
if results and "error" in results[0]:
|
| 592 |
+
return {"error": results[0]["error"]}
|
| 593 |
+
|
| 594 |
+
if serie:
|
| 595 |
+
results = [r for r in results if r.get("metadata", {}).get("serie") == serie]
|
| 596 |
+
|
| 597 |
+
registros = []
|
| 598 |
+
for r in results:
|
| 599 |
+
meta = r.get("metadata", {})
|
| 600 |
+
registros.append({
|
| 601 |
+
"serie": meta.get("serie"),
|
| 602 |
+
"nombre": meta.get("nombre"),
|
| 603 |
+
"fecha": meta.get("fecha"),
|
| 604 |
+
"valor": meta.get("valor"),
|
| 605 |
+
"relevancia": r.get("relevancia"),
|
| 606 |
+
})
|
| 607 |
+
|
| 608 |
+
return {
|
| 609 |
+
"fuente": "ChromaDB · NVIDIA NIM embeddings",
|
| 610 |
+
"modelo": MODEL_NAME,
|
| 611 |
+
"query": query,
|
| 612 |
+
"serie": serie or "todas",
|
| 613 |
+
"resultados": registros,
|
| 614 |
+
"total": len(registros),
|
| 615 |
+
}
|
| 616 |
+
|
| 617 |
+
|
| 618 |
+
def _search_historico_by_year(query: str, years: list, serie: str = None) -> dict:
|
| 619 |
+
path = os.path.join(DATA_PATH, "banxico_historico.json")
|
| 620 |
+
SERIES_NOMBRES = {
|
| 621 |
+
"SF43718": "Tipo de cambio USD/MXN",
|
| 622 |
+
"SF43783": "TIIE 28 días",
|
| 623 |
+
"SF61745": "Tasa objetivo Banxico",
|
| 624 |
+
"SP1": "Inflación IPC",
|
| 625 |
+
}
|
| 626 |
+
|
| 627 |
+
if not serie:
|
| 628 |
+
q = query.lower()
|
| 629 |
+
if any(k in q for k in ["tipo de cambio", "dólar", "dollar", "exchange", "peso", "sf43718"]):
|
| 630 |
+
serie = "SF43718"
|
| 631 |
+
elif any(k in q for k in ["tiie", "interbancaria", "sf43783"]):
|
| 632 |
+
serie = "SF43783"
|
| 633 |
+
elif any(k in q for k in ["tasa objetivo", "objetivo", "sf61745"]):
|
| 634 |
+
serie = "SF61745"
|
| 635 |
+
elif any(k in q for k in ["inflacion", "inflación", "ipc", "sp1"]):
|
| 636 |
+
serie = "SP1"
|
| 637 |
+
|
| 638 |
+
try:
|
| 639 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 640 |
+
data = json.load(f)
|
| 641 |
+
|
| 642 |
+
series_to_search = [serie] if serie else ["SF43718", "SF43783", "SF61745", "SP1"]
|
| 643 |
+
registros = []
|
| 644 |
+
|
| 645 |
+
for sid in series_to_search:
|
| 646 |
+
if sid not in data:
|
| 647 |
+
continue
|
| 648 |
+
for obs in data[sid]:
|
| 649 |
+
fecha = obs.get("fecha", "")
|
| 650 |
+
if any(year in fecha for year in years):
|
| 651 |
+
try:
|
| 652 |
+
registros.append({
|
| 653 |
+
"serie": sid,
|
| 654 |
+
"nombre": SERIES_NOMBRES.get(sid, sid),
|
| 655 |
+
"fecha": fecha,
|
| 656 |
+
"valor": str(obs.get("dato", "")),
|
| 657 |
+
"relevancia": 1.0,
|
| 658 |
+
})
|
| 659 |
+
except Exception:
|
| 660 |
+
pass
|
| 661 |
+
|
| 662 |
+
def sort_key(r):
|
| 663 |
+
f = r.get("fecha", "")
|
| 664 |
+
try:
|
| 665 |
+
for fmt in ["%d/%m/%Y", "%Y-%m-%d"]:
|
| 666 |
+
try:
|
| 667 |
+
return datetime.strptime(f, fmt)
|
| 668 |
+
except Exception:
|
| 669 |
+
pass
|
| 670 |
+
except Exception:
|
| 671 |
+
pass
|
| 672 |
+
return f
|
| 673 |
+
|
| 674 |
+
registros.sort(key=sort_key)
|
| 675 |
+
|
| 676 |
+
valores = []
|
| 677 |
+
for r in registros:
|
| 678 |
+
try:
|
| 679 |
+
valores.append(float(r["valor"]))
|
| 680 |
+
except Exception:
|
| 681 |
+
pass
|
| 682 |
+
|
| 683 |
+
stats = {}
|
| 684 |
+
if valores:
|
| 685 |
+
stats = {
|
| 686 |
+
"minimo": round(min(valores), 4),
|
| 687 |
+
"maximo": round(max(valores), 4),
|
| 688 |
+
"promedio": round(sum(valores)/len(valores), 4),
|
| 689 |
+
"primero": valores[0],
|
| 690 |
+
"ultimo": valores[-1],
|
| 691 |
+
}
|
| 692 |
+
|
| 693 |
+
return {
|
| 694 |
+
"fuente": "JSON directo · banxico_historico",
|
| 695 |
+
"modelo": "búsqueda_directa",
|
| 696 |
+
"query": query,
|
| 697 |
+
"serie": serie or "todas",
|
| 698 |
+
"años": years,
|
| 699 |
+
"resultados": registros,
|
| 700 |
+
"total": len(registros),
|
| 701 |
+
"estadisticas": stats,
|
| 702 |
+
"nota": f"Búsqueda directa por año(s): {', '.join(years)}",
|
| 703 |
+
}
|
| 704 |
+
|
| 705 |
+
except Exception as e:
|
| 706 |
+
return {"error": str(e)}
|
| 707 |
+
|
| 708 |
+
|
| 709 |
+
def search_all_sources(query: str, n_results: int = 3) -> dict:
|
| 710 |
+
collections = ["banxico_actual", "banxico_historico", "worldbank", "inegi", "ift"]
|
| 711 |
+
resultados = {}
|
| 712 |
+
|
| 713 |
+
for col in collections:
|
| 714 |
+
try:
|
| 715 |
+
col_obj = _get_collection(col)
|
| 716 |
+
if col_obj.count() == 0:
|
| 717 |
+
resultados[col] = {"status": "vacía"}
|
| 718 |
+
continue
|
| 719 |
+
results = search_chroma(query, col, n_results=n_results)
|
| 720 |
+
if results and "error" not in results[0]:
|
| 721 |
+
resultados[col] = {
|
| 722 |
+
"status": "ok",
|
| 723 |
+
"resultados": [
|
| 724 |
+
{"texto": r["texto"][:200], "metadata": r["metadata"], "relevancia": r["relevancia"]}
|
| 725 |
+
for r in results
|
| 726 |
+
]
|
| 727 |
+
}
|
| 728 |
+
else:
|
| 729 |
+
resultados[col] = {"status": "error", "detalle": results}
|
| 730 |
+
except Exception as e:
|
| 731 |
+
resultados[col] = {"status": "error", "detalle": str(e)}
|
| 732 |
+
|
| 733 |
+
return {
|
| 734 |
+
"fuente": "ChromaDB · NVIDIA NIM embeddings · búsqueda multi-colección",
|
| 735 |
+
"modelo": MODEL_NAME,
|
| 736 |
+
"query": query,
|
| 737 |
+
"fuentes": resultados,
|
| 738 |
+
}
|
| 739 |
+
|
| 740 |
+
|
| 741 |
+
# ─── Estado del índice ────────────────────────────────────────────────────────
|
| 742 |
+
|
| 743 |
+
def get_index_status() -> dict:
|
| 744 |
+
status = {}
|
| 745 |
+
for col_name in ["banxico_historico", "banxico_actual", "worldbank", "inegi", "ift"]:
|
| 746 |
+
try:
|
| 747 |
+
col = _get_collection(col_name)
|
| 748 |
+
count = col.count()
|
| 749 |
+
except Exception:
|
| 750 |
+
count = 0
|
| 751 |
+
idx = _index_status.get(col_name, {})
|
| 752 |
+
status[col_name] = {
|
| 753 |
+
"indexado": idx.get("indexed", False),
|
| 754 |
+
"documentos": count,
|
| 755 |
+
"ultimo_ts": idx.get("ts", "nunca"),
|
| 756 |
+
"hash": idx.get("hash", "—"),
|
| 757 |
+
}
|
| 758 |
+
return status
|
| 759 |
+
|
| 760 |
+
|
| 761 |
+
def init_chroma(force: bool = False) -> dict:
|
| 762 |
+
"""Punto de entrada único para inicializar ChromaDB al arrancar la app."""
|
| 763 |
+
return index_all(force=force)
|
| 764 |
+
|
| 765 |
+
|
| 766 |
+
if __name__ == "__main__":
|
| 767 |
+
print("Inicializando ChromaDB con NVIDIA NIM embeddings...")
|
| 768 |
+
result = init_chroma(force=True)
|
| 769 |
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
| 770 |
+
print("\nPrueba búsqueda IFT:")
|
| 771 |
+
r = search_chroma("operadores 5G México", "ift", n_results=3)
|
| 772 |
+
print(json.dumps(r, indent=2, ensure_ascii=False))
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def reset_collections() -> dict:
|
| 776 |
+
"""
|
| 777 |
+
Borra físicamente /data/chroma_db/ y limpia cache en memoria.
|
| 778 |
+
Necesario cuando cambia el modelo de embeddings (dimensión 384→1024).
|
| 779 |
+
"""
|
| 780 |
+
global _collections, _chroma_client
|
| 781 |
+
import shutil
|
| 782 |
+
|
| 783 |
+
# 1. Cerrar y destruir cliente ChromaDB en memoria
|
| 784 |
+
try:
|
| 785 |
+
if _chroma_client is not None:
|
| 786 |
+
del _chroma_client
|
| 787 |
+
except Exception:
|
| 788 |
+
pass
|
| 789 |
+
_chroma_client = None
|
| 790 |
+
_collections = {}
|
| 791 |
+
_index_status.clear()
|
| 792 |
+
|
| 793 |
+
# 2. Borrar archivos físicos en disco
|
| 794 |
+
if os.path.exists(CHROMA_PATH):
|
| 795 |
+
try:
|
| 796 |
+
shutil.rmtree(CHROMA_PATH)
|
| 797 |
+
except Exception as e:
|
| 798 |
+
return {"status": "error", "reason": f"No se pudo borrar {CHROMA_PATH}: {e}"}
|
| 799 |
+
|
| 800 |
+
# 3. Recrear carpeta vacía con permisos correctos
|
| 801 |
+
os.makedirs(CHROMA_PATH, exist_ok=True)
|
| 802 |
+
os.chmod(CHROMA_PATH, 0o777)
|
| 803 |
+
|
| 804 |
+
# 4. Crear cliente nuevo desde cero
|
| 805 |
+
import chromadb
|
| 806 |
+
_chroma_client = chromadb.EphemeralClient()
|
| 807 |
+
|
| 808 |
+
return {
|
| 809 |
+
"status": "ok",
|
| 810 |
+
"deleted": CHROMA_PATH,
|
| 811 |
+
"mensaje": "ChromaDB reseteado completamente. Cliente recreado. Listo para re-indexar.",
|
| 812 |
+
}
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
def init_chroma_fresh() -> dict:
|
| 816 |
+
"""
|
| 817 |
+
Borra físicamente ChromaDB y re-indexa desde cero con NVIDIA NIM embeddings.
|
| 818 |
+
Usar cuando se cambia el modelo de embeddings (ej: 384→1024 dimensiones).
|
| 819 |
+
"""
|
| 820 |
+
reset_result = reset_collections()
|
| 821 |
+
if reset_result.get("status") == "error":
|
| 822 |
+
return {"status": "error", "reset": reset_result}
|
| 823 |
+
index_result = index_all(force=True)
|
| 824 |
+
return {
|
| 825 |
+
"status": "ok" if index_result.get("errores", 1) == 0 else "partial",
|
| 826 |
+
"reset": reset_result,
|
| 827 |
+
"index": index_result,
|
| 828 |
+
}
|
modules/semantic_cache.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Capa 3 · Semantic Cache — Caché semántico sin Redis
|
| 3 |
+
====================================================
|
| 4 |
+
Implementa caché semántico en memoria usando embeddings NVIDIA NIM.
|
| 5 |
+
Evita llamadas repetidas a NVIDIA NIM para queries similares.
|
| 6 |
+
|
| 7 |
+
Sin Redis — usa dict en memoria + persistencia opcional en /data/cache.json.
|
| 8 |
+
Compatible con EphemeralClient de ChromaDB.
|
| 9 |
+
|
| 10 |
+
Flujo:
|
| 11 |
+
Query usuario
|
| 12 |
+
↓
|
| 13 |
+
[Generar embedding query] ~50ms
|
| 14 |
+
↓
|
| 15 |
+
[Buscar en caché por similitud coseno]
|
| 16 |
+
↓
|
| 17 |
+
Similitud > 0.92 → Retornar respuesta cacheada (~50ms total)
|
| 18 |
+
Similitud < 0.92 → Ejecutar pipeline completo → Guardar en caché
|
| 19 |
+
|
| 20 |
+
TTL por tipo de dato:
|
| 21 |
+
- Datos actuales (tasa, TC) → 6h (cambian diario)
|
| 22 |
+
- Datos históricos → 24h (no cambian)
|
| 23 |
+
- Telecomunicaciones IFT → 72h (trimestrales)
|
| 24 |
+
- Conceptos / Wikipedia → 7d (muy estables)
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import os
|
| 28 |
+
import json
|
| 29 |
+
import time
|
| 30 |
+
import hashlib
|
| 31 |
+
import requests
|
| 32 |
+
import numpy as np
|
| 33 |
+
from datetime import datetime, timedelta
|
| 34 |
+
from typing import Optional
|
| 35 |
+
|
| 36 |
+
CACHE_FILE = "/data/cache.json"
|
| 37 |
+
NVIDIA_API_KEY = os.getenv("NVIDIA_API_KEY", "")
|
| 38 |
+
NVIDIA_BASE = "https://integrate.api.nvidia.com/v1"
|
| 39 |
+
EMBED_MODEL = "nvidia/nv-embedqa-e5-v5"
|
| 40 |
+
|
| 41 |
+
# Umbral de similitud — queries con similitud > este valor se consideran iguales
|
| 42 |
+
SIMILARITY_THRESHOLD = 0.92
|
| 43 |
+
|
| 44 |
+
# TTL por categoría de query (segundos)
|
| 45 |
+
TTL_MAP = {
|
| 46 |
+
"actual": 6 * 3600, # 6h — tasa, TC, datos del día
|
| 47 |
+
"historico": 24 * 3600, # 24h — series históricas
|
| 48 |
+
"telco": 72 * 3600, # 72h — datos IFT trimestrales
|
| 49 |
+
"concepto": 7 * 86400, # 7d — definiciones, Wikipedia
|
| 50 |
+
"worldbank": 24 * 3600, # 24h — indicadores WB
|
| 51 |
+
"default": 12 * 3600, # 12h — todo lo demás
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
# ─── Cache en memoria ─────────────────────────────────────────────────────────
|
| 55 |
+
|
| 56 |
+
_cache: dict = {} # {hash: {embedding, response, timestamp, ttl, hits}}
|
| 57 |
+
_cache_loaded = False
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _get_ttl(query: str) -> int:
|
| 61 |
+
"""Determina el TTL apropiado según el tipo de query."""
|
| 62 |
+
q = query.lower()
|
| 63 |
+
if any(k in q for k in ["telcel", "at&t", "movistar", "ift", "5g", "telco", "telecomunicaciones"]):
|
| 64 |
+
return TTL_MAP["telco"]
|
| 65 |
+
if any(k in q for k in ["qué es", "que es", "cómo funciona", "como funciona", "definición", "what is"]):
|
| 66 |
+
return TTL_MAP["concepto"]
|
| 67 |
+
if any(k in q for k in ["pib", "exporta", "exportaciones", "importa", "deuda", "world bank", "remesas", "banco mundial"]):
|
| 68 |
+
return TTL_MAP["worldbank"]
|
| 69 |
+
if any(k in q for k in ["histórico", "historico", "2024", "2025", "2026", "tendencia", "promedio"]):
|
| 70 |
+
return TTL_MAP["historico"]
|
| 71 |
+
if any(k in q for k in ["hoy", "actual", "ahora", "tipo de cambio", "tasa", "tiie"]):
|
| 72 |
+
return TTL_MAP["actual"]
|
| 73 |
+
return TTL_MAP["default"]
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _cosine_similarity(a: list, b: list) -> float:
|
| 77 |
+
"""Calcula similitud coseno entre dos vectores."""
|
| 78 |
+
va = np.array(a, dtype=np.float32)
|
| 79 |
+
vb = np.array(b, dtype=np.float32)
|
| 80 |
+
norm_a = np.linalg.norm(va)
|
| 81 |
+
norm_b = np.linalg.norm(vb)
|
| 82 |
+
if norm_a == 0 or norm_b == 0:
|
| 83 |
+
return 0.0
|
| 84 |
+
return float(np.dot(va, vb) / (norm_a * norm_b))
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _get_embedding(text: str) -> Optional[list]:
|
| 88 |
+
"""Genera embedding para una query via NVIDIA NIM."""
|
| 89 |
+
if not NVIDIA_API_KEY:
|
| 90 |
+
return None
|
| 91 |
+
try:
|
| 92 |
+
r = requests.post(
|
| 93 |
+
f"{NVIDIA_BASE}/embeddings",
|
| 94 |
+
headers={
|
| 95 |
+
"Authorization": f"Bearer {NVIDIA_API_KEY}",
|
| 96 |
+
"Content-Type": "application/json",
|
| 97 |
+
},
|
| 98 |
+
json={
|
| 99 |
+
"model": EMBED_MODEL,
|
| 100 |
+
"input": [text],
|
| 101 |
+
"input_type": "query",
|
| 102 |
+
},
|
| 103 |
+
timeout=10,
|
| 104 |
+
)
|
| 105 |
+
r.raise_for_status()
|
| 106 |
+
return r.json()["data"][0]["embedding"]
|
| 107 |
+
except Exception:
|
| 108 |
+
return None
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _query_hash(query: str) -> str:
|
| 112 |
+
"""Hash MD5 de la query normalizada — para lookup rápido exacto."""
|
| 113 |
+
normalized = query.lower().strip()
|
| 114 |
+
return hashlib.md5(normalized.encode()).hexdigest()
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
# ─── Persistencia ────────────────────────────────────────────────────────────
|
| 118 |
+
|
| 119 |
+
def _load_cache():
|
| 120 |
+
"""Carga caché persistido desde /data/cache.json al arrancar."""
|
| 121 |
+
global _cache, _cache_loaded
|
| 122 |
+
if _cache_loaded:
|
| 123 |
+
return
|
| 124 |
+
try:
|
| 125 |
+
if os.path.exists(CACHE_FILE):
|
| 126 |
+
with open(CACHE_FILE, "r", encoding="utf-8") as f:
|
| 127 |
+
data = json.load(f)
|
| 128 |
+
# Filtrar entradas expiradas al cargar
|
| 129 |
+
now = time.time()
|
| 130 |
+
_cache = {
|
| 131 |
+
k: v for k, v in data.items()
|
| 132 |
+
if v.get("timestamp", 0) + v.get("ttl", 0) > now
|
| 133 |
+
}
|
| 134 |
+
except Exception:
|
| 135 |
+
_cache = {}
|
| 136 |
+
_cache_loaded = True
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _save_cache():
|
| 140 |
+
"""Persiste caché en /data/cache.json — solo entradas vigentes."""
|
| 141 |
+
try:
|
| 142 |
+
os.makedirs("/data", exist_ok=True)
|
| 143 |
+
now = time.time()
|
| 144 |
+
activos = {
|
| 145 |
+
k: {kk: vv for kk, vv in v.items() if kk != "embedding"}
|
| 146 |
+
# No guardamos embeddings (pesados) — se regeneran al cargar
|
| 147 |
+
for k, v in _cache.items()
|
| 148 |
+
if v.get("timestamp", 0) + v.get("ttl", 0) > now
|
| 149 |
+
}
|
| 150 |
+
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
| 151 |
+
json.dump(activos, f, ensure_ascii=False)
|
| 152 |
+
except Exception:
|
| 153 |
+
pass
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# ─── API pública ──────────────────────────────────────────────────────────────
|
| 157 |
+
|
| 158 |
+
def get_cached(query: str) -> Optional[str]:
|
| 159 |
+
"""
|
| 160 |
+
Busca una respuesta cacheada para la query.
|
| 161 |
+
|
| 162 |
+
Estrategia:
|
| 163 |
+
1. Lookup exacto por hash MD5 (O(1), ~0ms)
|
| 164 |
+
2. Si no hay match exacto → búsqueda semántica por similitud coseno
|
| 165 |
+
|
| 166 |
+
Returns:
|
| 167 |
+
str con la respuesta cacheada, o None si no hay match.
|
| 168 |
+
"""
|
| 169 |
+
_load_cache()
|
| 170 |
+
now = time.time()
|
| 171 |
+
|
| 172 |
+
# Paso 1: lookup exacto
|
| 173 |
+
h = _query_hash(query)
|
| 174 |
+
if h in _cache:
|
| 175 |
+
entry = _cache[h]
|
| 176 |
+
if entry.get("timestamp", 0) + entry.get("ttl", 0) > now:
|
| 177 |
+
_cache[h]["hits"] = _cache[h].get("hits", 0) + 1
|
| 178 |
+
return entry["response"]
|
| 179 |
+
else:
|
| 180 |
+
del _cache[h] # expirado
|
| 181 |
+
|
| 182 |
+
# Paso 2: búsqueda semántica
|
| 183 |
+
if not NVIDIA_API_KEY or not _cache:
|
| 184 |
+
return None
|
| 185 |
+
|
| 186 |
+
query_emb = _get_embedding(query)
|
| 187 |
+
if not query_emb:
|
| 188 |
+
return None
|
| 189 |
+
|
| 190 |
+
best_sim = 0.0
|
| 191 |
+
best_entry = None
|
| 192 |
+
best_key = None
|
| 193 |
+
|
| 194 |
+
for key, entry in list(_cache.items()):
|
| 195 |
+
# Verificar TTL
|
| 196 |
+
if entry.get("timestamp", 0) + entry.get("ttl", 0) <= now:
|
| 197 |
+
del _cache[key]
|
| 198 |
+
continue
|
| 199 |
+
emb = entry.get("embedding")
|
| 200 |
+
if not emb:
|
| 201 |
+
continue
|
| 202 |
+
sim = _cosine_similarity(query_emb, emb)
|
| 203 |
+
if sim > best_sim:
|
| 204 |
+
best_sim = sim
|
| 205 |
+
best_entry = entry
|
| 206 |
+
best_key = key
|
| 207 |
+
|
| 208 |
+
if best_sim >= SIMILARITY_THRESHOLD and best_entry:
|
| 209 |
+
_cache[best_key]["hits"] = _cache[best_key].get("hits", 0) + 1
|
| 210 |
+
return best_entry["response"]
|
| 211 |
+
|
| 212 |
+
return None
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def set_cached(query: str, response: str, ttl: int = None):
|
| 216 |
+
"""
|
| 217 |
+
Guarda una respuesta en el caché.
|
| 218 |
+
|
| 219 |
+
Args:
|
| 220 |
+
query: Query del usuario
|
| 221 |
+
response: Respuesta generada por el pipeline
|
| 222 |
+
ttl: Time-to-live en segundos (None = auto-detectar)
|
| 223 |
+
"""
|
| 224 |
+
_load_cache()
|
| 225 |
+
|
| 226 |
+
if ttl is None:
|
| 227 |
+
ttl = _get_ttl(query)
|
| 228 |
+
|
| 229 |
+
# Generar embedding para búsqueda semántica futura
|
| 230 |
+
emb = _get_embedding(query)
|
| 231 |
+
|
| 232 |
+
h = _query_hash(query)
|
| 233 |
+
_cache[h] = {
|
| 234 |
+
"query": query,
|
| 235 |
+
"response": response,
|
| 236 |
+
"embedding": emb,
|
| 237 |
+
"timestamp": time.time(),
|
| 238 |
+
"ttl": ttl,
|
| 239 |
+
"hits": 0,
|
| 240 |
+
"created": datetime.now().isoformat(),
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
# Persistir si el caché tiene >20 entradas nuevas (batch saves)
|
| 244 |
+
if len(_cache) % 20 == 0:
|
| 245 |
+
_save_cache()
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def invalidate_cache(pattern: str = None):
|
| 249 |
+
"""
|
| 250 |
+
Invalida entradas del caché.
|
| 251 |
+
pattern=None → invalida TODO el caché.
|
| 252 |
+
pattern='telco' → invalida solo queries relacionadas con telco.
|
| 253 |
+
"""
|
| 254 |
+
global _cache
|
| 255 |
+
if pattern is None:
|
| 256 |
+
_cache = {}
|
| 257 |
+
_save_cache()
|
| 258 |
+
return
|
| 259 |
+
|
| 260 |
+
keys_to_del = [
|
| 261 |
+
k for k, v in _cache.items()
|
| 262 |
+
if pattern.lower() in v.get("query", "").lower()
|
| 263 |
+
]
|
| 264 |
+
for k in keys_to_del:
|
| 265 |
+
del _cache[k]
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def get_cache_stats() -> dict:
|
| 269 |
+
"""Retorna estadísticas del caché para la UI de administración."""
|
| 270 |
+
_load_cache()
|
| 271 |
+
now = time.time()
|
| 272 |
+
activo = {k: v for k, v in _cache.items()
|
| 273 |
+
if v.get("timestamp", 0) + v.get("ttl", 0) > now}
|
| 274 |
+
total_hits = sum(v.get("hits", 0) for v in activo.values())
|
| 275 |
+
|
| 276 |
+
return {
|
| 277 |
+
"entradas_activas": len(activo),
|
| 278 |
+
"total_hits": total_hits,
|
| 279 |
+
"hit_rate_pct": round(total_hits / max(len(activo), 1), 1),
|
| 280 |
+
"threshold": SIMILARITY_THRESHOLD,
|
| 281 |
+
"ttl_config": TTL_MAP,
|
| 282 |
+
"entradas_recientes": [
|
| 283 |
+
{
|
| 284 |
+
"query": v.get("query", "")[:60],
|
| 285 |
+
"hits": v.get("hits", 0),
|
| 286 |
+
"ttl_h": round(v.get("ttl", 0) / 3600, 1),
|
| 287 |
+
"created": v.get("created", "")[:16],
|
| 288 |
+
}
|
| 289 |
+
for v in sorted(activo.values(),
|
| 290 |
+
key=lambda x: x.get("hits", 0), reverse=True)[:5]
|
| 291 |
+
],
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
if __name__ == "__main__":
|
| 296 |
+
print("🧪 Test Semantic Cache\n")
|
| 297 |
+
|
| 298 |
+
# Test TTL detection
|
| 299 |
+
queries = [
|
| 300 |
+
("¿Cuál es el tipo de cambio hoy?", "actual"),
|
| 301 |
+
("¿Cuál fue el TC en enero 2025?", "historico"),
|
| 302 |
+
("¿Cuál es la participación de Telcel?", "telco"),
|
| 303 |
+
("¿Qué es el PIB?", "concepto"),
|
| 304 |
+
("¿Cuánto exporta México?", "worldbank"),
|
| 305 |
+
]
|
| 306 |
+
for q, expected in queries:
|
| 307 |
+
ttl = _get_ttl(q)
|
| 308 |
+
category = next(k for k, v in TTL_MAP.items() if v == ttl)
|
| 309 |
+
ok = "✅" if category == expected else "❌"
|
| 310 |
+
print(f"{ok} '{q[:45]}' → {category} ({ttl//3600}h TTL)")
|
| 311 |
+
|
| 312 |
+
print("\nTest cosine similarity:")
|
| 313 |
+
a = [1.0, 0.0, 0.0]
|
| 314 |
+
b = [1.0, 0.0, 0.0]
|
| 315 |
+
c = [0.0, 1.0, 0.0]
|
| 316 |
+
print(f" identical vectors: {_cosine_similarity(a, b):.3f} (expected: 1.0)")
|
| 317 |
+
print(f" orthogonal vectors: {_cosine_similarity(a, c):.3f} (expected: 0.0)")
|