"""Owned, durable execution traces. Never records tokens, keys or SQL parameters.""" import contextvars import functools import hashlib import inspect import threading import time import uuid from contextlib import contextmanager from datetime import datetime, timezone from sqlalchemy import event _active = contextvars.ContextVar('workos_trace', default=None) _parent = contextvars.ContextVar('workos_span', default=None) _secret_names = {'authorization','password','access_token','refresh_token','api_key','key','headers','secret','token','user','u','self'} def snapshot(value, depth=0): if depth > 6: return '[内容层级已折叠]' if isinstance(value, bytes): return {'bytes':len(value), 'sha256':hashlib.sha256(value).hexdigest()} if isinstance(value, str): return value[:16384] + ('…[截断]' if len(value)>16384 else '') if value is None or isinstance(value,(int,float,bool)): return value if isinstance(value,dict): return {str(k):snapshot(v,depth+1) for k,v in list(value.items())[:60] if str(k).lower() not in _secret_names} if isinstance(value,(list,tuple)): return [snapshot(v,depth+1) for v in value[:40]] if hasattr(value,'document_id'): return {k:getattr(value,k,None) for k in ('document_id','version','base_id','status','error_code','sha256')} return {'type':type(value).__name__} class Trace: def __init__(self, Session, operation, *, trace_id=None, user=None, workspace=None, run_id=None): self.Session=Session self.id=trace_id or 'trace-'+uuid.uuid4().hex self.operation=operation self.user=user self.workspace=workspace self.run_id=run_id self.started=time.perf_counter() self.started_at=datetime.now(timezone.utc).isoformat() self.spans=[] self.database_groups={} self.omitted_database_executions=0 self.chunks=[] self.lock=threading.RLock() self.closed=False def __enter__(self): self.token=_active.set(self) self.parent_token=_parent.set(None) return self def start(self,module,operation,input_data=None,parent=None): with self.lock: if self.closed:return None parent_id=parent if parent is not None else _parent.get() group=None if module=='Database': group=(parent_id,operation,str(input_data)) if group in self.database_groups: item=self.database_groups[group];item['executions']+=1 return item if len(self.spans)>=350: if module=='Database': self.omitted_database_executions+=1;return None victim=next((s for s in self.spans if s['module']=='Database' and s['status']=='ok'),None) if victim is None:return None self.omitted_database_executions+=victim.get('executions',1) self.spans.remove(victim) for k in [k for k,v in self.database_groups.items() if v is victim]:del self.database_groups[k] item={'span_id':'s-'+uuid.uuid4().hex[:12], 'parent_id':parent_id, 'module':module,'operation':operation,'status':'running','started_ms':round((time.perf_counter()-self.started)*1000,2), 'input':snapshot(input_data),'result':None,'error':None} self.spans.append(item) if group is not None: item['executions']=1;self.database_groups[group]=item return item def end(self,item,started,*,result=None,error=None): if item is None:return with self.lock: elapsed=(time.perf_counter()-started)*1000 item['duration_ms']=round(elapsed+ (item.get('duration_ms',0) if item['module']=='Database' else 0),2) item['status']='error' if error or item['status']=='error' else 'ok' item['result']=snapshot(result) if item['module']=='Database':item['result']={**(item['result'] or {}),'executions':item['executions'],'aggregation':'相同父阶段与SQL模板合并,参数未记录;耗时为各次执行之和。'} if error:item['error']={'code':getattr(error,'code',type(error).__name__),'message':getattr(error,'message',type(error).__name__)} def finish(self,status='ok'): if self.closed:return self.closed=True if not self.user:return if self.omitted_database_executions and self.spans: self.spans[0]['trace_limits']={'omitted_database_executions':self.omitted_database_executions,'notice':'SQL 明细达到上限;优先保留业务模块和模型调用阶段。知识块账本独立分页保留。'} from KnowledgeService.models import TraceRecord,TraceChunk # Trace storage must not recursively trace itself. token=_active.set(None) try: with self.Session.begin() as db: db.add(TraceRecord(id=self.id,tenant=self.user['tenant_id'],owner=self.user['user_id'],workspace=self.workspace, run_id=self.run_id,operation=self.operation,status=status,duration_ms=int((time.perf_counter()-self.started)*1000), started_at=self.started_at,spans=self.spans)) for ordinal,chunk in enumerate(self.chunks,1):db.add(TraceChunk(trace_id=self.id,ordinal=ordinal,body=chunk)) finally: _active.reset(token) def __exit__(self,typ,exc,tb): try:self.finish('error' if exc else 'ok') finally: _parent.reset(self.parent_token) _active.reset(self.token) def scope(user=None,workspace=None,run_id=None): trace=_active.get() if trace and not trace.closed: if user:trace.user=user if workspace:trace.workspace=workspace if run_id:trace.run_id=run_id def chunk_ledger(rows): trace=_active.get() if trace and not trace.closed: with trace.lock:trace.chunks.extend(rows) def trace_chunks(workspace,knowledge,user,wid,tid,after=0,limit=50): from sqlalchemy import select from KnowledgeService.models import TraceRecord,TraceChunk workspace.assert_access(user,wid) from APIGateway.app.errors import GatewayError with workspace.Session() as db: t=db.get(TraceRecord,tid) if not t or (t.tenant,t.owner,t.workspace)!=(user['tenant_id'],user['user_id'],wid):raise GatewayError(404,'RESOURCE_NOT_FOUND','链路不存在或无权查看。') rows=db.scalars(select(TraceChunk).where(TraceChunk.trace_id==tid,TraceChunk.ordinal>after).order_by(TraceChunk.ordinal).limit(limit+1)).all() items=[] for row in rows[:limit]: try:knowledge.citation(user,row.body['citation_id'],wid) except Exception as e: if getattr(e,'status',None)==404:continue raise items.append({**row.body,'ledger_ordinal':row.ordinal}) return {'items':items,'next_cursor':rows[limit-1].ordinal if len(rows)>limit else None,'scope':'当前用户仍获授权的知识块'} @contextmanager def span(module,operation,input_data=None): trace=_active.get() started=time.perf_counter() item=trace.start(module,operation,input_data) if trace else None token=_parent.set(item['span_id']) if item else None result={} try: yield result except Exception as exc: if trace:trace.end(item,started,error=exc) raise else: if trace:trace.end(item,started,result=result.get('value',result)) finally: if token:_parent.reset(token) def traced(module,operation=None): def decorate(fn): signature=inspect.signature(fn) def inputs(args,kwargs): values=signature.bind(*args,**kwargs).arguments # user identity comes only from verified Gateway dependency, never from payload. user=values.get('user') or values.get('u') wid=values.get('wid') or values.get('workspace_id') scope(user, wid) return {k:v for k,v in values.items() if k not in _secret_names} if inspect.iscoroutinefunction(fn): @functools.wraps(fn) async def async_call(*args,**kwargs): if not _active.get():return await fn(*args,**kwargs) with span(module,operation or fn.__name__,inputs(args,kwargs)) as output: value=await fn(*args,**kwargs) output['value']=value return value return async_call @functools.wraps(fn) def call(*args,**kwargs): if not _active.get():return fn(*args,**kwargs) with span(module,operation or fn.__name__,inputs(args,kwargs)) as output: value=fn(*args,**kwargs) output['value']=value return value return call return decorate def instrument_engine(engine): if getattr(engine,'_workos_traced',False):return engine._workos_traced=True @event.listens_for(engine,'before_cursor_execute') def before(conn,cursor,statement,parameters,context,executemany): trace=_active.get() if trace and not trace.closed: # SQL text uses bound parameters; parameter values are deliberately excluded. sql=' '.join(statement.split()) item=trace.start('Database',sql.split(' ',1)[0],{'statement':sql[:1000],'parameters':'已隐藏','dialect':engine.dialect.name}) context._workos_span=(trace,item,time.perf_counter()) @event.listens_for(engine,'after_cursor_execute') def after(conn,cursor,statement,parameters,context,executemany): value=getattr(context,'_workos_span',None) if value: trace,item,started=value trace.end(item,started,result={'row_count':cursor.rowcount,'executed':True}) @event.listens_for(engine,'handle_error') def error(context): value=getattr(context.execution_context,'_workos_span',None) if value: trace,item,started=value trace.end(item,started,error=context.original_exception) def trace_rows(workspace,user,wid): from sqlalchemy import select,func from KnowledgeService.models import TraceRecord,TraceChunk workspace.assert_access(user,wid) with workspace.Session() as db: rows=db.scalars(select(TraceRecord).where(TraceRecord.tenant==user['tenant_id'],TraceRecord.owner==user['user_id'], TraceRecord.workspace==wid).order_by(TraceRecord.started_at.desc()).limit(60)).all() counts=dict(db.execute(select(TraceChunk.trace_id,func.count()).where(TraceChunk.trace_id.in_([r.id for r in rows])).group_by(TraceChunk.trace_id)).all()) return {'items':[{**{k:getattr(r,k) for k in ('id','run_id','operation','status','duration_ms','started_at','spans')},'chunk_count':counts.get(r.id,0)} for r in rows], 'limits':{'max_traces':60,'max_spans_per_trace':350,'max_text_chars':16384}, 'notice':'记录真实模块调用与SQL执行;不记录鉴权令牌、模型密钥和SQL参数。长内容按上限截断。'}