mx-data-api / modules /observability.py
Fercor95mx's picture
Upload 6 files
24431ef verified
Raw History Blame Contribute Delete
11.9 kB
"""
Capa 3 Β· Observabilidad β€” MΓ©tricas del sistema
===============================================
Registra latencia, tokens, costo estimado y rutas del pipeline.
MΓ©tricas por query:
- Latencia total y por etapa (clasificador, tools, sΓ­ntesis, validador)
- Tokens input/output por modelo
- Costo estimado USD (basado en pricing NVIDIA NIM)
- Ruta tomada (SIMPLE/NUMÉRICA/NARRATIVA)
- Tool calls ejecutadas
- Cache hit/miss
- Errores del Critic-Loop detectados
Persistencia: /data/metrics.jsonl (append-only, mΓ‘x 1000 lΓ­neas)
UI: tabla en AdministraciΓ³n + resumen de sesiΓ³n
"""
import os
import json
import time
import threading
from datetime import datetime
from collections import defaultdict
METRICS_FILE = "/data/metrics.jsonl"
MAX_LINES = 1000
# Pricing NVIDIA NIM (USD por 1M tokens β€” estimado)
PRICING = {
"meta/llama-3.1-8b-instruct": {"input": 0.20, "output": 0.20},
"meta/llama-3.1-70b-instruct": {"input": 0.99, "output": 0.99},
"mistralai/mixtral-8x7b-instruct-v0.1": {"input": 0.60, "output": 0.60},
"nvidia/nv-embedqa-e5-v5": {"input": 0.06, "output": 0.0},
"nvidia/nv-rerankqa-mistral-4b-v3": {"input": 0.08, "output": 0.0},
"default": {"input": 0.50, "output": 0.50},
}
_lock = threading.Lock()
_session_metrics = [] # mΓ©tricas de la sesiΓ³n actual en RAM
# ─── Contexto de mediciΓ³n ─────────────────────────────────────────────────────
class QueryTrace:
"""
Contexto de trazabilidad para una query completa.
Uso:
trace = QueryTrace(question)
trace.start_stage("clasificador")
... cΓ³digo ...
trace.end_stage("clasificador")
trace.record_tokens("llama-70b", input=150, output=300)
trace.set_route("NARRATIVA")
trace.add_tool("read_banxico")
metrics = trace.finish()
"""
def __init__(self, question: str):
self.question = question[:120]
self.start_time = time.time()
self.stages = {}
self.tokens = defaultdict(lambda: {"input": 0, "output": 0})
self.route = "UNKNOWN"
self.tools_used = []
self.cache_hit = False
self.critic_errors = 0
self.critic_iters = 0
self._stage_start = {}
def start_stage(self, stage: str):
self._stage_start[stage] = time.time()
def end_stage(self, stage: str):
if stage in self._stage_start:
ms = int((time.time() - self._stage_start[stage]) * 1000)
self.stages[stage] = ms
def record_tokens(self, model: str, input_tokens: int = 0, output_tokens: int = 0):
self.tokens[model]["input"] += input_tokens
self.tokens[model]["output"] += output_tokens
def set_route(self, route: str):
self.route = route
def add_tool(self, tool_name: str):
if tool_name not in self.tools_used:
self.tools_used.append(tool_name)
def set_cache_hit(self, hit: bool):
self.cache_hit = hit
def set_critic(self, errors: int, iterations: int):
self.critic_errors = errors
self.critic_iters = iterations
def finish(self) -> dict:
total_ms = int((time.time() - self.start_time) * 1000)
# Calcular costo estimado
total_cost_usd = 0.0
token_summary = {}
for model, counts in self.tokens.items():
pricing = PRICING.get(model, PRICING["default"])
cost = (counts["input"] / 1_000_000 * pricing["input"] +
counts["output"] / 1_000_000 * pricing["output"])
total_cost_usd += cost
token_summary[model] = {
"input": counts["input"],
"output": counts["output"],
"cost_usd": round(cost, 6),
}
metric = {
"ts": datetime.now().isoformat(),
"question": self.question,
"route": self.route,
"total_ms": total_ms,
"stages_ms": self.stages,
"tokens": token_summary,
"cost_usd": round(total_cost_usd, 6),
"tools_used": self.tools_used,
"cache_hit": self.cache_hit,
"critic_errors": self.critic_errors,
"critic_iters": self.critic_iters,
}
record_metric(metric)
return metric
# ─── Persistencia ─────────────────────────────────────────────────────────────
def record_metric(metric: dict):
"""Guarda mΓ©trica en /data/metrics.jsonl y en RAM."""
global _session_metrics
with _lock:
_session_metrics.append(metric)
_session_metrics = _session_metrics[-500:] # mΓ‘x 500 en RAM
try:
os.makedirs("/data", exist_ok=True)
with open(METRICS_FILE, "a", encoding="utf-8") as f:
f.write(json.dumps(metric, ensure_ascii=False) + "\n")
_trim_metrics_file()
except Exception:
pass
def _trim_metrics_file():
"""Mantiene el archivo de mΓ©tricas en mΓ‘ximo MAX_LINES lΓ­neas."""
try:
if not os.path.exists(METRICS_FILE):
return
with open(METRICS_FILE, "r") as f:
lines = f.readlines()
if len(lines) > MAX_LINES:
with open(METRICS_FILE, "w") as f:
f.writelines(lines[-MAX_LINES:])
except Exception:
pass
def load_metrics(last_n: int = 100) -> list:
"""Carga las ΓΊltimas N mΓ©tricas del archivo."""
try:
if not os.path.exists(METRICS_FILE):
return []
with open(METRICS_FILE, "r", encoding="utf-8") as f:
lines = f.readlines()
metrics = []
for line in lines[-last_n:]:
try:
metrics.append(json.loads(line.strip()))
except Exception:
pass
return metrics
except Exception:
return []
# ─── AnΓ‘lisis y estadΓ­sticas ──────────────────────────────────────────────────
def get_session_stats() -> dict:
"""EstadΓ­sticas de la sesiΓ³n actual (desde el ΓΊltimo arranque)."""
if not _session_metrics:
return {"queries": 0, "message": "Sin mΓ©tricas en esta sesiΓ³n"}
queries = len(_session_metrics)
total_ms = sum(m.get("total_ms", 0) for m in _session_metrics)
cache_hits = sum(1 for m in _session_metrics if m.get("cache_hit"))
total_cost = sum(m.get("cost_usd", 0) for m in _session_metrics)
critic_total = sum(m.get("critic_errors", 0) for m in _session_metrics)
# Latencia por ruta
by_route = defaultdict(list)
for m in _session_metrics:
by_route[m.get("route", "UNKNOWN")].append(m.get("total_ms", 0))
route_stats = {
route: {
"count": len(times),
"avg_ms": int(sum(times) / len(times)),
"min_ms": min(times),
"max_ms": max(times),
}
for route, times in by_route.items()
}
# Tools mΓ‘s usadas
tool_counts = defaultdict(int)
for m in _session_metrics:
for t in m.get("tools_used", []):
tool_counts[t] += 1
return {
"queries": queries,
"avg_latencia_ms": int(total_ms / queries) if queries else 0,
"cache_hits": cache_hits,
"cache_hit_rate": f"{cache_hits/queries*100:.0f}%" if queries else "0%",
"costo_total_usd": round(total_cost, 4),
"costo_por_query": round(total_cost / queries, 6) if queries else 0,
"critic_errors": critic_total,
"por_ruta": route_stats,
"tools_mas_usadas": dict(sorted(tool_counts.items(),
key=lambda x: x[1], reverse=True)[:5]),
"ultima_query": _session_metrics[-1].get("question", "")[:60] if _session_metrics else "",
}
def get_metrics_for_ui(last_n: int = 20) -> list:
"""
Retorna mΓ©tricas formateadas para la tabla de administraciΓ³n.
"""
metrics = load_metrics(last_n)
rows = []
for m in reversed(metrics):
ts = m.get("ts", "")[:16].replace("T", " ")
q = m.get("question", "")[:50]
route = m.get("route", "?")
ms = m.get("total_ms", 0)
cost = m.get("cost_usd", 0)
cache = "βœ… HIT" if m.get("cache_hit") else "❌ MISS"
tools = ", ".join(m.get("tools_used", []))
rows.append({
"ts": ts,
"query": q,
"route": route,
"ms": ms,
"cost": f"${cost:.5f}",
"cache": cache,
"tools": tools,
})
return rows
# ─── Decorador para medir funciones ──────────────────────────────────────────
def measure(stage_name: str, trace: QueryTrace = None):
"""
Decorador/context manager para medir latencia de una etapa.
Uso: with measure("clasificador", trace): ...
"""
class _Ctx:
def __enter__(self):
if trace:
trace.start_stage(stage_name)
self._t = time.time()
return self
def __exit__(self, *args):
if trace:
trace.end_stage(stage_name)
return _Ctx()
# ─── IntegraciΓ³n con call_model ───────────────────────────────────────────────
def wrap_call_model(original_call_model, trace: QueryTrace = None):
"""
Wrapper para call_model que registra tokens automΓ‘ticamente.
Uso en agent_orchestrator.py:
result = wrap_call_model(call_model, trace)(model, messages, max_tokens)
"""
def wrapped(model: str, messages: list, max_tokens: int = 700):
t0 = time.time()
result, error = original_call_model(model, messages, max_tokens)
ms = int((time.time() - t0) * 1000)
if trace and result:
# EstimaciΓ³n de tokens (4 chars β‰ˆ 1 token)
input_tokens = sum(len(m.get("content", "")) for m in messages) // 4
output_tokens = len(result) // 4
trace.record_tokens(model, input_tokens, output_tokens)
stage = model.split("/")[-1][:20]
trace.stages[stage] = ms
return result, error
return wrapped
if __name__ == "__main__":
print("πŸ§ͺ Test Observabilidad\n")
# Simular una query completa
trace = QueryTrace("ΒΏCuΓ‘l es la tasa objetivo del Banxico hoy?")
trace.start_stage("clasificador")
time.sleep(0.05)
trace.end_stage("clasificador")
trace.set_route("SIMPLE")
trace.add_tool("read_banxico")
trace.record_tokens("meta/llama-3.1-8b-instruct", 50, 10)
trace.record_tokens("meta/llama-3.1-70b-instruct", 200, 150)
trace.set_cache_hit(False)
trace.set_critic(errors=0, iterations=1)
metric = trace.finish()
print(f"βœ… Query registrada:")
print(f" Ruta: {metric['route']}")
print(f" Latencia total: {metric['total_ms']}ms")
print(f" Costo estimado: ${metric['cost_usd']:.6f} USD")
print(f" Tools: {metric['tools_used']}")
print(f" Stages: {metric['stages_ms']}")
stats = get_session_stats()
print(f"\nπŸ“Š Stats sesiΓ³n:")
print(f" Queries: {stats['queries']}")
print(f" Avg latencia: {stats['avg_latencia_ms']}ms")
print(f" Costo total: ${stats['costo_total_usd']} USD")