File size: 11,866 Bytes
24431ef
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
"""
Capa 3 Β· Observabilidad β€” MΓ©tricas del sistema
===============================================
Registra latencia, tokens, costo estimado y rutas del pipeline.

MΓ©tricas por query:
  - Latencia total y por etapa (clasificador, tools, sΓ­ntesis, validador)
  - Tokens input/output por modelo
  - Costo estimado USD (basado en pricing NVIDIA NIM)
  - Ruta tomada (SIMPLE/NUMÉRICA/NARRATIVA)
  - Tool calls ejecutadas
  - Cache hit/miss
  - Errores del Critic-Loop detectados

Persistencia: /data/metrics.jsonl (append-only, mΓ‘x 1000 lΓ­neas)
UI: tabla en AdministraciΓ³n + resumen de sesiΓ³n
"""

import os
import json
import time
import threading
from datetime import datetime
from collections import defaultdict

METRICS_FILE = "/data/metrics.jsonl"
MAX_LINES    = 1000

# Pricing NVIDIA NIM (USD por 1M tokens β€” estimado)
PRICING = {
    "meta/llama-3.1-8b-instruct":           {"input": 0.20,  "output": 0.20},
    "meta/llama-3.1-70b-instruct":          {"input": 0.99,  "output": 0.99},
    "mistralai/mixtral-8x7b-instruct-v0.1": {"input": 0.60,  "output": 0.60},
    "nvidia/nv-embedqa-e5-v5":              {"input": 0.06,  "output": 0.0},
    "nvidia/nv-rerankqa-mistral-4b-v3":     {"input": 0.08,  "output": 0.0},
    "default":                               {"input": 0.50,  "output": 0.50},
}

_lock        = threading.Lock()
_session_metrics = []   # mΓ©tricas de la sesiΓ³n actual en RAM


# ─── Contexto de mediciΓ³n ─────────────────────────────────────────────────────

class QueryTrace:
    """
    Contexto de trazabilidad para una query completa.
    Uso:
        trace = QueryTrace(question)
        trace.start_stage("clasificador")
        ... cΓ³digo ...
        trace.end_stage("clasificador")
        trace.record_tokens("llama-70b", input=150, output=300)
        trace.set_route("NARRATIVA")
        trace.add_tool("read_banxico")
        metrics = trace.finish()
    """

    def __init__(self, question: str):
        self.question     = question[:120]
        self.start_time   = time.time()
        self.stages       = {}
        self.tokens       = defaultdict(lambda: {"input": 0, "output": 0})
        self.route        = "UNKNOWN"
        self.tools_used   = []
        self.cache_hit    = False
        self.critic_errors = 0
        self.critic_iters  = 0
        self._stage_start  = {}

    def start_stage(self, stage: str):
        self._stage_start[stage] = time.time()

    def end_stage(self, stage: str):
        if stage in self._stage_start:
            ms = int((time.time() - self._stage_start[stage]) * 1000)
            self.stages[stage] = ms

    def record_tokens(self, model: str, input_tokens: int = 0, output_tokens: int = 0):
        self.tokens[model]["input"]  += input_tokens
        self.tokens[model]["output"] += output_tokens

    def set_route(self, route: str):
        self.route = route

    def add_tool(self, tool_name: str):
        if tool_name not in self.tools_used:
            self.tools_used.append(tool_name)

    def set_cache_hit(self, hit: bool):
        self.cache_hit = hit

    def set_critic(self, errors: int, iterations: int):
        self.critic_errors = errors
        self.critic_iters  = iterations

    def finish(self) -> dict:
        total_ms = int((time.time() - self.start_time) * 1000)

        # Calcular costo estimado
        total_cost_usd = 0.0
        token_summary  = {}
        for model, counts in self.tokens.items():
            pricing   = PRICING.get(model, PRICING["default"])
            cost      = (counts["input"]  / 1_000_000 * pricing["input"] +
                         counts["output"] / 1_000_000 * pricing["output"])
            total_cost_usd += cost
            token_summary[model] = {
                "input":    counts["input"],
                "output":   counts["output"],
                "cost_usd": round(cost, 6),
            }

        metric = {
            "ts":           datetime.now().isoformat(),
            "question":     self.question,
            "route":        self.route,
            "total_ms":     total_ms,
            "stages_ms":    self.stages,
            "tokens":       token_summary,
            "cost_usd":     round(total_cost_usd, 6),
            "tools_used":   self.tools_used,
            "cache_hit":    self.cache_hit,
            "critic_errors": self.critic_errors,
            "critic_iters": self.critic_iters,
        }

        record_metric(metric)
        return metric


# ─── Persistencia ─────────────────────────────────────────────────────────────

def record_metric(metric: dict):
    """Guarda mΓ©trica en /data/metrics.jsonl y en RAM."""
    global _session_metrics
    with _lock:
        _session_metrics.append(metric)
        _session_metrics = _session_metrics[-500:]  # mΓ‘x 500 en RAM

    try:
        os.makedirs("/data", exist_ok=True)
        with open(METRICS_FILE, "a", encoding="utf-8") as f:
            f.write(json.dumps(metric, ensure_ascii=False) + "\n")
        _trim_metrics_file()
    except Exception:
        pass


def _trim_metrics_file():
    """Mantiene el archivo de mΓ©tricas en mΓ‘ximo MAX_LINES lΓ­neas."""
    try:
        if not os.path.exists(METRICS_FILE):
            return
        with open(METRICS_FILE, "r") as f:
            lines = f.readlines()
        if len(lines) > MAX_LINES:
            with open(METRICS_FILE, "w") as f:
                f.writelines(lines[-MAX_LINES:])
    except Exception:
        pass


def load_metrics(last_n: int = 100) -> list:
    """Carga las ΓΊltimas N mΓ©tricas del archivo."""
    try:
        if not os.path.exists(METRICS_FILE):
            return []
        with open(METRICS_FILE, "r", encoding="utf-8") as f:
            lines = f.readlines()
        metrics = []
        for line in lines[-last_n:]:
            try:
                metrics.append(json.loads(line.strip()))
            except Exception:
                pass
        return metrics
    except Exception:
        return []


# ─── AnΓ‘lisis y estadΓ­sticas ──────────────────────────────────────────────────

def get_session_stats() -> dict:
    """EstadΓ­sticas de la sesiΓ³n actual (desde el ΓΊltimo arranque)."""
    if not _session_metrics:
        return {"queries": 0, "message": "Sin mΓ©tricas en esta sesiΓ³n"}

    queries      = len(_session_metrics)
    total_ms     = sum(m.get("total_ms", 0) for m in _session_metrics)
    cache_hits   = sum(1 for m in _session_metrics if m.get("cache_hit"))
    total_cost   = sum(m.get("cost_usd", 0) for m in _session_metrics)
    critic_total = sum(m.get("critic_errors", 0) for m in _session_metrics)

    # Latencia por ruta
    by_route = defaultdict(list)
    for m in _session_metrics:
        by_route[m.get("route", "UNKNOWN")].append(m.get("total_ms", 0))

    route_stats = {
        route: {
            "count":      len(times),
            "avg_ms":     int(sum(times) / len(times)),
            "min_ms":     min(times),
            "max_ms":     max(times),
        }
        for route, times in by_route.items()
    }

    # Tools mΓ‘s usadas
    tool_counts = defaultdict(int)
    for m in _session_metrics:
        for t in m.get("tools_used", []):
            tool_counts[t] += 1

    return {
        "queries":          queries,
        "avg_latencia_ms":  int(total_ms / queries) if queries else 0,
        "cache_hits":       cache_hits,
        "cache_hit_rate":   f"{cache_hits/queries*100:.0f}%" if queries else "0%",
        "costo_total_usd":  round(total_cost, 4),
        "costo_por_query":  round(total_cost / queries, 6) if queries else 0,
        "critic_errors":    critic_total,
        "por_ruta":         route_stats,
        "tools_mas_usadas": dict(sorted(tool_counts.items(),
                                        key=lambda x: x[1], reverse=True)[:5]),
        "ultima_query":     _session_metrics[-1].get("question", "")[:60] if _session_metrics else "",
    }


def get_metrics_for_ui(last_n: int = 20) -> list:
    """
    Retorna mΓ©tricas formateadas para la tabla de administraciΓ³n.
    """
    metrics = load_metrics(last_n)
    rows    = []
    for m in reversed(metrics):
        ts    = m.get("ts", "")[:16].replace("T", " ")
        q     = m.get("question", "")[:50]
        route = m.get("route", "?")
        ms    = m.get("total_ms", 0)
        cost  = m.get("cost_usd", 0)
        cache = "βœ… HIT" if m.get("cache_hit") else "❌ MISS"
        tools = ", ".join(m.get("tools_used", []))
        rows.append({
            "ts":     ts,
            "query":  q,
            "route":  route,
            "ms":     ms,
            "cost":   f"${cost:.5f}",
            "cache":  cache,
            "tools":  tools,
        })
    return rows


# ─── Decorador para medir funciones ──────────────────────────────────────────

def measure(stage_name: str, trace: QueryTrace = None):
    """
    Decorador/context manager para medir latencia de una etapa.
    Uso: with measure("clasificador", trace): ...
    """
    class _Ctx:
        def __enter__(self):
            if trace:
                trace.start_stage(stage_name)
            self._t = time.time()
            return self

        def __exit__(self, *args):
            if trace:
                trace.end_stage(stage_name)

    return _Ctx()


# ─── IntegraciΓ³n con call_model ───────────────────────────────────────────────

def wrap_call_model(original_call_model, trace: QueryTrace = None):
    """
    Wrapper para call_model que registra tokens automΓ‘ticamente.
    Uso en agent_orchestrator.py:
        result = wrap_call_model(call_model, trace)(model, messages, max_tokens)
    """
    def wrapped(model: str, messages: list, max_tokens: int = 700):
        t0      = time.time()
        result, error = original_call_model(model, messages, max_tokens)
        ms      = int((time.time() - t0) * 1000)

        if trace and result:
            # EstimaciΓ³n de tokens (4 chars β‰ˆ 1 token)
            input_tokens  = sum(len(m.get("content", "")) for m in messages) // 4
            output_tokens = len(result) // 4
            trace.record_tokens(model, input_tokens, output_tokens)
            stage = model.split("/")[-1][:20]
            trace.stages[stage] = ms

        return result, error
    return wrapped


if __name__ == "__main__":
    print("πŸ§ͺ Test Observabilidad\n")

    # Simular una query completa
    trace = QueryTrace("ΒΏCuΓ‘l es la tasa objetivo del Banxico hoy?")
    trace.start_stage("clasificador")
    time.sleep(0.05)
    trace.end_stage("clasificador")
    trace.set_route("SIMPLE")
    trace.add_tool("read_banxico")
    trace.record_tokens("meta/llama-3.1-8b-instruct", 50, 10)
    trace.record_tokens("meta/llama-3.1-70b-instruct", 200, 150)
    trace.set_cache_hit(False)
    trace.set_critic(errors=0, iterations=1)

    metric = trace.finish()
    print(f"βœ… Query registrada:")
    print(f"   Ruta: {metric['route']}")
    print(f"   Latencia total: {metric['total_ms']}ms")
    print(f"   Costo estimado: ${metric['cost_usd']:.6f} USD")
    print(f"   Tools: {metric['tools_used']}")
    print(f"   Stages: {metric['stages_ms']}")

    stats = get_session_stats()
    print(f"\nπŸ“Š Stats sesiΓ³n:")
    print(f"   Queries: {stats['queries']}")
    print(f"   Avg latencia: {stats['avg_latencia_ms']}ms")
    print(f"   Costo total: ${stats['costo_total_usd']} USD")