File size: 11,428 Bytes
daa4d56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
"""Owned, durable execution traces. Never records tokens, keys or SQL parameters."""
import contextvars
import functools
import hashlib
import inspect
import threading
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from sqlalchemy import event

_active = contextvars.ContextVar('workos_trace', default=None)
_parent = contextvars.ContextVar('workos_span', default=None)
_secret_names = {'authorization','password','access_token','refresh_token','api_key','key','headers','secret','token','user','u','self'}


def snapshot(value, depth=0):
    if depth > 6:
        return '[内容层级已折叠]'
    if isinstance(value, bytes):
        return {'bytes':len(value), 'sha256':hashlib.sha256(value).hexdigest()}
    if isinstance(value, str):
        return value[:16384] + ('…[截断]' if len(value)>16384 else '')
    if value is None or isinstance(value,(int,float,bool)):
        return value
    if isinstance(value,dict):
        return {str(k):snapshot(v,depth+1) for k,v in list(value.items())[:60] if str(k).lower() not in _secret_names}
    if isinstance(value,(list,tuple)):
        return [snapshot(v,depth+1) for v in value[:40]]
    if hasattr(value,'document_id'):
        return {k:getattr(value,k,None) for k in ('document_id','version','base_id','status','error_code','sha256')}
    return {'type':type(value).__name__}


class Trace:
    def __init__(self, Session, operation, *, trace_id=None, user=None, workspace=None, run_id=None):
        self.Session=Session
        self.id=trace_id or 'trace-'+uuid.uuid4().hex
        self.operation=operation
        self.user=user
        self.workspace=workspace
        self.run_id=run_id
        self.started=time.perf_counter()
        self.started_at=datetime.now(timezone.utc).isoformat()
        self.spans=[]
        self.database_groups={}
        self.omitted_database_executions=0
        self.chunks=[]
        self.lock=threading.RLock()
        self.closed=False

    def __enter__(self):
        self.token=_active.set(self)
        self.parent_token=_parent.set(None)
        return self

    def start(self,module,operation,input_data=None,parent=None):
        with self.lock:
            if self.closed:return None
            parent_id=parent if parent is not None else _parent.get()
            group=None
            if module=='Database':
                group=(parent_id,operation,str(input_data))
                if group in self.database_groups:
                    item=self.database_groups[group];item['executions']+=1
                    return item
            if len(self.spans)>=350:
                if module=='Database':
                    self.omitted_database_executions+=1;return None
                victim=next((s for s in self.spans if s['module']=='Database' and s['status']=='ok'),None)
                if victim is None:return None
                self.omitted_database_executions+=victim.get('executions',1)
                self.spans.remove(victim)
                for k in [k for k,v in self.database_groups.items() if v is victim]:del self.database_groups[k]
            item={'span_id':'s-'+uuid.uuid4().hex[:12], 'parent_id':parent_id,
                  'module':module,'operation':operation,'status':'running','started_ms':round((time.perf_counter()-self.started)*1000,2),
                  'input':snapshot(input_data),'result':None,'error':None}
            self.spans.append(item)
            if group is not None:
                item['executions']=1;self.database_groups[group]=item
            return item

    def end(self,item,started,*,result=None,error=None):
        if item is None:return
        with self.lock:
            elapsed=(time.perf_counter()-started)*1000
            item['duration_ms']=round(elapsed+ (item.get('duration_ms',0) if item['module']=='Database' else 0),2)
            item['status']='error' if error or item['status']=='error' else 'ok'
            item['result']=snapshot(result)
            if item['module']=='Database':item['result']={**(item['result'] or {}),'executions':item['executions'],'aggregation':'相同父阶段与SQL模板合并,参数未记录;耗时为各次执行之和。'}
            if error:item['error']={'code':getattr(error,'code',type(error).__name__),'message':getattr(error,'message',type(error).__name__)}

    def finish(self,status='ok'):
        if self.closed:return
        self.closed=True
        if not self.user:return
        if self.omitted_database_executions and self.spans:
            self.spans[0]['trace_limits']={'omitted_database_executions':self.omitted_database_executions,'notice':'SQL 明细达到上限;优先保留业务模块和模型调用阶段。知识块账本独立分页保留。'}
        from KnowledgeService.models import TraceRecord,TraceChunk
        # Trace storage must not recursively trace itself.
        token=_active.set(None)
        try:
            with self.Session.begin() as db:
                db.add(TraceRecord(id=self.id,tenant=self.user['tenant_id'],owner=self.user['user_id'],workspace=self.workspace,
                    run_id=self.run_id,operation=self.operation,status=status,duration_ms=int((time.perf_counter()-self.started)*1000),
                    started_at=self.started_at,spans=self.spans))
                for ordinal,chunk in enumerate(self.chunks,1):db.add(TraceChunk(trace_id=self.id,ordinal=ordinal,body=chunk))
        finally:
            _active.reset(token)

    def __exit__(self,typ,exc,tb):
        try:self.finish('error' if exc else 'ok')
        finally:
            _parent.reset(self.parent_token)
            _active.reset(self.token)


def scope(user=None,workspace=None,run_id=None):
    trace=_active.get()
    if trace and not trace.closed:
        if user:trace.user=user
        if workspace:trace.workspace=workspace
        if run_id:trace.run_id=run_id

def chunk_ledger(rows):
    trace=_active.get()
    if trace and not trace.closed:
        with trace.lock:trace.chunks.extend(rows)

def trace_chunks(workspace,knowledge,user,wid,tid,after=0,limit=50):
    from sqlalchemy import select
    from KnowledgeService.models import TraceRecord,TraceChunk
    workspace.assert_access(user,wid)
    from APIGateway.app.errors import GatewayError
    with workspace.Session() as db:
        t=db.get(TraceRecord,tid)
        if not t or (t.tenant,t.owner,t.workspace)!=(user['tenant_id'],user['user_id'],wid):raise GatewayError(404,'RESOURCE_NOT_FOUND','链路不存在或无权查看。')
        rows=db.scalars(select(TraceChunk).where(TraceChunk.trace_id==tid,TraceChunk.ordinal>after).order_by(TraceChunk.ordinal).limit(limit+1)).all()
        items=[]
        for row in rows[:limit]:
            try:knowledge.citation(user,row.body['citation_id'],wid)
            except Exception as e:
                if getattr(e,'status',None)==404:continue
                raise
            items.append({**row.body,'ledger_ordinal':row.ordinal})
        return {'items':items,'next_cursor':rows[limit-1].ordinal if len(rows)>limit else None,'scope':'当前用户仍获授权的知识块'}


@contextmanager
def span(module,operation,input_data=None):
    trace=_active.get()
    started=time.perf_counter()
    item=trace.start(module,operation,input_data) if trace else None
    token=_parent.set(item['span_id']) if item else None
    result={}
    try:
        yield result
    except Exception as exc:
        if trace:trace.end(item,started,error=exc)
        raise
    else:
        if trace:trace.end(item,started,result=result.get('value',result))
    finally:
        if token:_parent.reset(token)


def traced(module,operation=None):
    def decorate(fn):
        signature=inspect.signature(fn)
        def inputs(args,kwargs):
            values=signature.bind(*args,**kwargs).arguments
            # user identity comes only from verified Gateway dependency, never from payload.
            user=values.get('user') or values.get('u')
            wid=values.get('wid') or values.get('workspace_id')
            scope(user, wid)
            return {k:v for k,v in values.items() if k not in _secret_names}
        if inspect.iscoroutinefunction(fn):
            @functools.wraps(fn)
            async def async_call(*args,**kwargs):
                if not _active.get():return await fn(*args,**kwargs)
                with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
                    value=await fn(*args,**kwargs)
                    output['value']=value
                    return value
            return async_call
        @functools.wraps(fn)
        def call(*args,**kwargs):
            if not _active.get():return fn(*args,**kwargs)
            with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
                value=fn(*args,**kwargs)
                output['value']=value
                return value
        return call
    return decorate


def instrument_engine(engine):
    if getattr(engine,'_workos_traced',False):return
    engine._workos_traced=True
    @event.listens_for(engine,'before_cursor_execute')
    def before(conn,cursor,statement,parameters,context,executemany):
        trace=_active.get()
        if trace and not trace.closed:
            # SQL text uses bound parameters; parameter values are deliberately excluded.
            sql=' '.join(statement.split())
            item=trace.start('Database',sql.split(' ',1)[0],{'statement':sql[:1000],'parameters':'已隐藏','dialect':engine.dialect.name})
            context._workos_span=(trace,item,time.perf_counter())
    @event.listens_for(engine,'after_cursor_execute')
    def after(conn,cursor,statement,parameters,context,executemany):
        value=getattr(context,'_workos_span',None)
        if value:
            trace,item,started=value
            trace.end(item,started,result={'row_count':cursor.rowcount,'executed':True})
    @event.listens_for(engine,'handle_error')
    def error(context):
        value=getattr(context.execution_context,'_workos_span',None)
        if value:
            trace,item,started=value
            trace.end(item,started,error=context.original_exception)


def trace_rows(workspace,user,wid):
    from sqlalchemy import select,func
    from KnowledgeService.models import TraceRecord,TraceChunk
    workspace.assert_access(user,wid)
    with workspace.Session() as db:
        rows=db.scalars(select(TraceRecord).where(TraceRecord.tenant==user['tenant_id'],TraceRecord.owner==user['user_id'],
            TraceRecord.workspace==wid).order_by(TraceRecord.started_at.desc()).limit(60)).all()
        counts=dict(db.execute(select(TraceChunk.trace_id,func.count()).where(TraceChunk.trace_id.in_([r.id for r in rows])).group_by(TraceChunk.trace_id)).all())
        return {'items':[{**{k:getattr(r,k) for k in ('id','run_id','operation','status','duration_ms','started_at','spans')},'chunk_count':counts.get(r.id,0)} for r in rows],
                'limits':{'max_traces':60,'max_spans_per_trace':350,'max_text_chars':16384},
                'notice':'记录真实模块调用与SQL执行;不记录鉴权令牌、模型密钥和SQL参数。长内容按上限截断。'}