""" Capa 3 · Observabilidad — Métricas del sistema =============================================== Registra latencia, tokens, costo estimado y rutas del pipeline. Métricas por query: - Latencia total y por etapa (clasificador, tools, síntesis, validador) - Tokens input/output por modelo - Costo estimado USD (basado en pricing NVIDIA NIM) - Ruta tomada (SIMPLE/NUMÉRICA/NARRATIVA) - Tool calls ejecutadas - Cache hit/miss - Errores del Critic-Loop detectados Persistencia: /data/metrics.jsonl (append-only, máx 1000 líneas) UI: tabla en Administración + resumen de sesión """ import os import json import time import threading from datetime import datetime from collections import defaultdict METRICS_FILE = "/data/metrics.jsonl" MAX_LINES = 1000 # Pricing NVIDIA NIM (USD por 1M tokens — estimado) PRICING = { "meta/llama-3.1-8b-instruct": {"input": 0.20, "output": 0.20}, "meta/llama-3.1-70b-instruct": {"input": 0.99, "output": 0.99}, "mistralai/mixtral-8x7b-instruct-v0.1": {"input": 0.60, "output": 0.60}, "nvidia/nv-embedqa-e5-v5": {"input": 0.06, "output": 0.0}, "nvidia/nv-rerankqa-mistral-4b-v3": {"input": 0.08, "output": 0.0}, "default": {"input": 0.50, "output": 0.50}, } _lock = threading.Lock() _session_metrics = [] # métricas de la sesión actual en RAM # ─── Contexto de medición ───────────────────────────────────────────────────── class QueryTrace: """ Contexto de trazabilidad para una query completa. Uso: trace = QueryTrace(question) trace.start_stage("clasificador") ... código ... trace.end_stage("clasificador") trace.record_tokens("llama-70b", input=150, output=300) trace.set_route("NARRATIVA") trace.add_tool("read_banxico") metrics = trace.finish() """ def __init__(self, question: str): self.question = question[:120] self.start_time = time.time() self.stages = {} self.tokens = defaultdict(lambda: {"input": 0, "output": 0}) self.route = "UNKNOWN" self.tools_used = [] self.cache_hit = False self.critic_errors = 0 self.critic_iters = 0 self._stage_start = {} def start_stage(self, stage: str): self._stage_start[stage] = time.time() def end_stage(self, stage: str): if stage in self._stage_start: ms = int((time.time() - self._stage_start[stage]) * 1000) self.stages[stage] = ms def record_tokens(self, model: str, input_tokens: int = 0, output_tokens: int = 0): self.tokens[model]["input"] += input_tokens self.tokens[model]["output"] += output_tokens def set_route(self, route: str): self.route = route def add_tool(self, tool_name: str): if tool_name not in self.tools_used: self.tools_used.append(tool_name) def set_cache_hit(self, hit: bool): self.cache_hit = hit def set_critic(self, errors: int, iterations: int): self.critic_errors = errors self.critic_iters = iterations def finish(self) -> dict: total_ms = int((time.time() - self.start_time) * 1000) # Calcular costo estimado total_cost_usd = 0.0 token_summary = {} for model, counts in self.tokens.items(): pricing = PRICING.get(model, PRICING["default"]) cost = (counts["input"] / 1_000_000 * pricing["input"] + counts["output"] / 1_000_000 * pricing["output"]) total_cost_usd += cost token_summary[model] = { "input": counts["input"], "output": counts["output"], "cost_usd": round(cost, 6), } metric = { "ts": datetime.now().isoformat(), "question": self.question, "route": self.route, "total_ms": total_ms, "stages_ms": self.stages, "tokens": token_summary, "cost_usd": round(total_cost_usd, 6), "tools_used": self.tools_used, "cache_hit": self.cache_hit, "critic_errors": self.critic_errors, "critic_iters": self.critic_iters, } record_metric(metric) return metric # ─── Persistencia ───────────────────────────────────────────────────────────── def record_metric(metric: dict): """Guarda métrica en /data/metrics.jsonl y en RAM.""" global _session_metrics with _lock: _session_metrics.append(metric) _session_metrics = _session_metrics[-500:] # máx 500 en RAM try: os.makedirs("/data", exist_ok=True) with open(METRICS_FILE, "a", encoding="utf-8") as f: f.write(json.dumps(metric, ensure_ascii=False) + "\n") _trim_metrics_file() except Exception: pass def _trim_metrics_file(): """Mantiene el archivo de métricas en máximo MAX_LINES líneas.""" try: if not os.path.exists(METRICS_FILE): return with open(METRICS_FILE, "r") as f: lines = f.readlines() if len(lines) > MAX_LINES: with open(METRICS_FILE, "w") as f: f.writelines(lines[-MAX_LINES:]) except Exception: pass def load_metrics(last_n: int = 100) -> list: """Carga las últimas N métricas del archivo.""" try: if not os.path.exists(METRICS_FILE): return [] with open(METRICS_FILE, "r", encoding="utf-8") as f: lines = f.readlines() metrics = [] for line in lines[-last_n:]: try: metrics.append(json.loads(line.strip())) except Exception: pass return metrics except Exception: return [] # ─── Análisis y estadísticas ────────────────────────────────────────────────── def get_session_stats() -> dict: """Estadísticas de la sesión actual (desde el último arranque).""" if not _session_metrics: return {"queries": 0, "message": "Sin métricas en esta sesión"} queries = len(_session_metrics) total_ms = sum(m.get("total_ms", 0) for m in _session_metrics) cache_hits = sum(1 for m in _session_metrics if m.get("cache_hit")) total_cost = sum(m.get("cost_usd", 0) for m in _session_metrics) critic_total = sum(m.get("critic_errors", 0) for m in _session_metrics) # Latencia por ruta by_route = defaultdict(list) for m in _session_metrics: by_route[m.get("route", "UNKNOWN")].append(m.get("total_ms", 0)) route_stats = { route: { "count": len(times), "avg_ms": int(sum(times) / len(times)), "min_ms": min(times), "max_ms": max(times), } for route, times in by_route.items() } # Tools más usadas tool_counts = defaultdict(int) for m in _session_metrics: for t in m.get("tools_used", []): tool_counts[t] += 1 return { "queries": queries, "avg_latencia_ms": int(total_ms / queries) if queries else 0, "cache_hits": cache_hits, "cache_hit_rate": f"{cache_hits/queries*100:.0f}%" if queries else "0%", "costo_total_usd": round(total_cost, 4), "costo_por_query": round(total_cost / queries, 6) if queries else 0, "critic_errors": critic_total, "por_ruta": route_stats, "tools_mas_usadas": dict(sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)[:5]), "ultima_query": _session_metrics[-1].get("question", "")[:60] if _session_metrics else "", } def get_metrics_for_ui(last_n: int = 20) -> list: """ Retorna métricas formateadas para la tabla de administración. """ metrics = load_metrics(last_n) rows = [] for m in reversed(metrics): ts = m.get("ts", "")[:16].replace("T", " ") q = m.get("question", "")[:50] route = m.get("route", "?") ms = m.get("total_ms", 0) cost = m.get("cost_usd", 0) cache = "✅ HIT" if m.get("cache_hit") else "❌ MISS" tools = ", ".join(m.get("tools_used", [])) rows.append({ "ts": ts, "query": q, "route": route, "ms": ms, "cost": f"${cost:.5f}", "cache": cache, "tools": tools, }) return rows # ─── Decorador para medir funciones ────────────────────────────────────────── def measure(stage_name: str, trace: QueryTrace = None): """ Decorador/context manager para medir latencia de una etapa. Uso: with measure("clasificador", trace): ... """ class _Ctx: def __enter__(self): if trace: trace.start_stage(stage_name) self._t = time.time() return self def __exit__(self, *args): if trace: trace.end_stage(stage_name) return _Ctx() # ─── Integración con call_model ─────────────────────────────────────────────── def wrap_call_model(original_call_model, trace: QueryTrace = None): """ Wrapper para call_model que registra tokens automáticamente. Uso en agent_orchestrator.py: result = wrap_call_model(call_model, trace)(model, messages, max_tokens) """ def wrapped(model: str, messages: list, max_tokens: int = 700): t0 = time.time() result, error = original_call_model(model, messages, max_tokens) ms = int((time.time() - t0) * 1000) if trace and result: # Estimación de tokens (4 chars ≈ 1 token) input_tokens = sum(len(m.get("content", "")) for m in messages) // 4 output_tokens = len(result) // 4 trace.record_tokens(model, input_tokens, output_tokens) stage = model.split("/")[-1][:20] trace.stages[stage] = ms return result, error return wrapped if __name__ == "__main__": print("🧪 Test Observabilidad\n") # Simular una query completa trace = QueryTrace("¿Cuál es la tasa objetivo del Banxico hoy?") trace.start_stage("clasificador") time.sleep(0.05) trace.end_stage("clasificador") trace.set_route("SIMPLE") trace.add_tool("read_banxico") trace.record_tokens("meta/llama-3.1-8b-instruct", 50, 10) trace.record_tokens("meta/llama-3.1-70b-instruct", 200, 150) trace.set_cache_hit(False) trace.set_critic(errors=0, iterations=1) metric = trace.finish() print(f"✅ Query registrada:") print(f" Ruta: {metric['route']}") print(f" Latencia total: {metric['total_ms']}ms") print(f" Costo estimado: ${metric['cost_usd']:.6f} USD") print(f" Tools: {metric['tools_used']}") print(f" Stages: {metric['stages_ms']}") stats = get_session_stats() print(f"\n📊 Stats sesión:") print(f" Queries: {stats['queries']}") print(f" Avg latencia: {stats['avg_latencia_ms']}ms") print(f" Costo total: ${stats['costo_total_usd']} USD")