Spaces:
Sleeping
Sleeping
Download modules/observability.py from Fercor95mx/mx-data-api: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/Fercor95mx/mx-data-api/resolve/main/modules/observability.py
- Command line
-
hf download hf://spaces/Fercor95mx/mx-data-api/modules/observability.py
-
curl -L -o observability.py https://huggingface.co/spaces/Fercor95mx/mx-data-api/resolve/main/modules/observability.py
11.9 kB
| """ | |
| Capa 3 Β· Observabilidad β MΓ©tricas del sistema | |
| =============================================== | |
| Registra latencia, tokens, costo estimado y rutas del pipeline. | |
| MΓ©tricas por query: | |
| - Latencia total y por etapa (clasificador, tools, sΓntesis, validador) | |
| - Tokens input/output por modelo | |
| - Costo estimado USD (basado en pricing NVIDIA NIM) | |
| - Ruta tomada (SIMPLE/NUMΓRICA/NARRATIVA) | |
| - Tool calls ejecutadas | |
| - Cache hit/miss | |
| - Errores del Critic-Loop detectados | |
| Persistencia: /data/metrics.jsonl (append-only, mΓ‘x 1000 lΓneas) | |
| UI: tabla en AdministraciΓ³n + resumen de sesiΓ³n | |
| """ | |
| import os | |
| import json | |
| import time | |
| import threading | |
| from datetime import datetime | |
| from collections import defaultdict | |
| METRICS_FILE = "/data/metrics.jsonl" | |
| MAX_LINES = 1000 | |
| # Pricing NVIDIA NIM (USD por 1M tokens β estimado) | |
| PRICING = { | |
| "meta/llama-3.1-8b-instruct": {"input": 0.20, "output": 0.20}, | |
| "meta/llama-3.1-70b-instruct": {"input": 0.99, "output": 0.99}, | |
| "mistralai/mixtral-8x7b-instruct-v0.1": {"input": 0.60, "output": 0.60}, | |
| "nvidia/nv-embedqa-e5-v5": {"input": 0.06, "output": 0.0}, | |
| "nvidia/nv-rerankqa-mistral-4b-v3": {"input": 0.08, "output": 0.0}, | |
| "default": {"input": 0.50, "output": 0.50}, | |
| } | |
| _lock = threading.Lock() | |
| _session_metrics = [] # mΓ©tricas de la sesiΓ³n actual en RAM | |
| # βββ Contexto de mediciΓ³n βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| class QueryTrace: | |
| """ | |
| Contexto de trazabilidad para una query completa. | |
| Uso: | |
| trace = QueryTrace(question) | |
| trace.start_stage("clasificador") | |
| ... cΓ³digo ... | |
| trace.end_stage("clasificador") | |
| trace.record_tokens("llama-70b", input=150, output=300) | |
| trace.set_route("NARRATIVA") | |
| trace.add_tool("read_banxico") | |
| metrics = trace.finish() | |
| """ | |
| def __init__(self, question: str): | |
| self.question = question[:120] | |
| self.start_time = time.time() | |
| self.stages = {} | |
| self.tokens = defaultdict(lambda: {"input": 0, "output": 0}) | |
| self.route = "UNKNOWN" | |
| self.tools_used = [] | |
| self.cache_hit = False | |
| self.critic_errors = 0 | |
| self.critic_iters = 0 | |
| self._stage_start = {} | |
| def start_stage(self, stage: str): | |
| self._stage_start[stage] = time.time() | |
| def end_stage(self, stage: str): | |
| if stage in self._stage_start: | |
| ms = int((time.time() - self._stage_start[stage]) * 1000) | |
| self.stages[stage] = ms | |
| def record_tokens(self, model: str, input_tokens: int = 0, output_tokens: int = 0): | |
| self.tokens[model]["input"] += input_tokens | |
| self.tokens[model]["output"] += output_tokens | |
| def set_route(self, route: str): | |
| self.route = route | |
| def add_tool(self, tool_name: str): | |
| if tool_name not in self.tools_used: | |
| self.tools_used.append(tool_name) | |
| def set_cache_hit(self, hit: bool): | |
| self.cache_hit = hit | |
| def set_critic(self, errors: int, iterations: int): | |
| self.critic_errors = errors | |
| self.critic_iters = iterations | |
| def finish(self) -> dict: | |
| total_ms = int((time.time() - self.start_time) * 1000) | |
| # Calcular costo estimado | |
| total_cost_usd = 0.0 | |
| token_summary = {} | |
| for model, counts in self.tokens.items(): | |
| pricing = PRICING.get(model, PRICING["default"]) | |
| cost = (counts["input"] / 1_000_000 * pricing["input"] + | |
| counts["output"] / 1_000_000 * pricing["output"]) | |
| total_cost_usd += cost | |
| token_summary[model] = { | |
| "input": counts["input"], | |
| "output": counts["output"], | |
| "cost_usd": round(cost, 6), | |
| } | |
| metric = { | |
| "ts": datetime.now().isoformat(), | |
| "question": self.question, | |
| "route": self.route, | |
| "total_ms": total_ms, | |
| "stages_ms": self.stages, | |
| "tokens": token_summary, | |
| "cost_usd": round(total_cost_usd, 6), | |
| "tools_used": self.tools_used, | |
| "cache_hit": self.cache_hit, | |
| "critic_errors": self.critic_errors, | |
| "critic_iters": self.critic_iters, | |
| } | |
| record_metric(metric) | |
| return metric | |
| # βββ Persistencia βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def record_metric(metric: dict): | |
| """Guarda mΓ©trica en /data/metrics.jsonl y en RAM.""" | |
| global _session_metrics | |
| with _lock: | |
| _session_metrics.append(metric) | |
| _session_metrics = _session_metrics[-500:] # mΓ‘x 500 en RAM | |
| try: | |
| os.makedirs("/data", exist_ok=True) | |
| with open(METRICS_FILE, "a", encoding="utf-8") as f: | |
| f.write(json.dumps(metric, ensure_ascii=False) + "\n") | |
| _trim_metrics_file() | |
| except Exception: | |
| pass | |
| def _trim_metrics_file(): | |
| """Mantiene el archivo de mΓ©tricas en mΓ‘ximo MAX_LINES lΓneas.""" | |
| try: | |
| if not os.path.exists(METRICS_FILE): | |
| return | |
| with open(METRICS_FILE, "r") as f: | |
| lines = f.readlines() | |
| if len(lines) > MAX_LINES: | |
| with open(METRICS_FILE, "w") as f: | |
| f.writelines(lines[-MAX_LINES:]) | |
| except Exception: | |
| pass | |
| def load_metrics(last_n: int = 100) -> list: | |
| """Carga las ΓΊltimas N mΓ©tricas del archivo.""" | |
| try: | |
| if not os.path.exists(METRICS_FILE): | |
| return [] | |
| with open(METRICS_FILE, "r", encoding="utf-8") as f: | |
| lines = f.readlines() | |
| metrics = [] | |
| for line in lines[-last_n:]: | |
| try: | |
| metrics.append(json.loads(line.strip())) | |
| except Exception: | |
| pass | |
| return metrics | |
| except Exception: | |
| return [] | |
| # βββ AnΓ‘lisis y estadΓsticas ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def get_session_stats() -> dict: | |
| """EstadΓsticas de la sesiΓ³n actual (desde el ΓΊltimo arranque).""" | |
| if not _session_metrics: | |
| return {"queries": 0, "message": "Sin mΓ©tricas en esta sesiΓ³n"} | |
| queries = len(_session_metrics) | |
| total_ms = sum(m.get("total_ms", 0) for m in _session_metrics) | |
| cache_hits = sum(1 for m in _session_metrics if m.get("cache_hit")) | |
| total_cost = sum(m.get("cost_usd", 0) for m in _session_metrics) | |
| critic_total = sum(m.get("critic_errors", 0) for m in _session_metrics) | |
| # Latencia por ruta | |
| by_route = defaultdict(list) | |
| for m in _session_metrics: | |
| by_route[m.get("route", "UNKNOWN")].append(m.get("total_ms", 0)) | |
| route_stats = { | |
| route: { | |
| "count": len(times), | |
| "avg_ms": int(sum(times) / len(times)), | |
| "min_ms": min(times), | |
| "max_ms": max(times), | |
| } | |
| for route, times in by_route.items() | |
| } | |
| # Tools mΓ‘s usadas | |
| tool_counts = defaultdict(int) | |
| for m in _session_metrics: | |
| for t in m.get("tools_used", []): | |
| tool_counts[t] += 1 | |
| return { | |
| "queries": queries, | |
| "avg_latencia_ms": int(total_ms / queries) if queries else 0, | |
| "cache_hits": cache_hits, | |
| "cache_hit_rate": f"{cache_hits/queries*100:.0f}%" if queries else "0%", | |
| "costo_total_usd": round(total_cost, 4), | |
| "costo_por_query": round(total_cost / queries, 6) if queries else 0, | |
| "critic_errors": critic_total, | |
| "por_ruta": route_stats, | |
| "tools_mas_usadas": dict(sorted(tool_counts.items(), | |
| key=lambda x: x[1], reverse=True)[:5]), | |
| "ultima_query": _session_metrics[-1].get("question", "")[:60] if _session_metrics else "", | |
| } | |
| def get_metrics_for_ui(last_n: int = 20) -> list: | |
| """ | |
| Retorna mΓ©tricas formateadas para la tabla de administraciΓ³n. | |
| """ | |
| metrics = load_metrics(last_n) | |
| rows = [] | |
| for m in reversed(metrics): | |
| ts = m.get("ts", "")[:16].replace("T", " ") | |
| q = m.get("question", "")[:50] | |
| route = m.get("route", "?") | |
| ms = m.get("total_ms", 0) | |
| cost = m.get("cost_usd", 0) | |
| cache = "β HIT" if m.get("cache_hit") else "β MISS" | |
| tools = ", ".join(m.get("tools_used", [])) | |
| rows.append({ | |
| "ts": ts, | |
| "query": q, | |
| "route": route, | |
| "ms": ms, | |
| "cost": f"${cost:.5f}", | |
| "cache": cache, | |
| "tools": tools, | |
| }) | |
| return rows | |
| # βββ Decorador para medir funciones ββββββββββββββββββββββββββββββββββββββββββ | |
| def measure(stage_name: str, trace: QueryTrace = None): | |
| """ | |
| Decorador/context manager para medir latencia de una etapa. | |
| Uso: with measure("clasificador", trace): ... | |
| """ | |
| class _Ctx: | |
| def __enter__(self): | |
| if trace: | |
| trace.start_stage(stage_name) | |
| self._t = time.time() | |
| return self | |
| def __exit__(self, *args): | |
| if trace: | |
| trace.end_stage(stage_name) | |
| return _Ctx() | |
| # βββ IntegraciΓ³n con call_model βββββββββββββββββββββββββββββββββββββββββββββββ | |
| def wrap_call_model(original_call_model, trace: QueryTrace = None): | |
| """ | |
| Wrapper para call_model que registra tokens automΓ‘ticamente. | |
| Uso en agent_orchestrator.py: | |
| result = wrap_call_model(call_model, trace)(model, messages, max_tokens) | |
| """ | |
| def wrapped(model: str, messages: list, max_tokens: int = 700): | |
| t0 = time.time() | |
| result, error = original_call_model(model, messages, max_tokens) | |
| ms = int((time.time() - t0) * 1000) | |
| if trace and result: | |
| # EstimaciΓ³n de tokens (4 chars β 1 token) | |
| input_tokens = sum(len(m.get("content", "")) for m in messages) // 4 | |
| output_tokens = len(result) // 4 | |
| trace.record_tokens(model, input_tokens, output_tokens) | |
| stage = model.split("/")[-1][:20] | |
| trace.stages[stage] = ms | |
| return result, error | |
| return wrapped | |
| if __name__ == "__main__": | |
| print("π§ͺ Test Observabilidad\n") | |
| # Simular una query completa | |
| trace = QueryTrace("ΒΏCuΓ‘l es la tasa objetivo del Banxico hoy?") | |
| trace.start_stage("clasificador") | |
| time.sleep(0.05) | |
| trace.end_stage("clasificador") | |
| trace.set_route("SIMPLE") | |
| trace.add_tool("read_banxico") | |
| trace.record_tokens("meta/llama-3.1-8b-instruct", 50, 10) | |
| trace.record_tokens("meta/llama-3.1-70b-instruct", 200, 150) | |
| trace.set_cache_hit(False) | |
| trace.set_critic(errors=0, iterations=1) | |
| metric = trace.finish() | |
| print(f"β Query registrada:") | |
| print(f" Ruta: {metric['route']}") | |
| print(f" Latencia total: {metric['total_ms']}ms") | |
| print(f" Costo estimado: ${metric['cost_usd']:.6f} USD") | |
| print(f" Tools: {metric['tools_used']}") | |
| print(f" Stages: {metric['stages_ms']}") | |
| stats = get_session_stats() | |
| print(f"\nπ Stats sesiΓ³n:") | |
| print(f" Queries: {stats['queries']}") | |
| print(f" Avg latencia: {stats['avg_latencia_ms']}ms") | |
| print(f" Costo total: ${stats['costo_total_usd']} USD") |