Spaces:
Sleeping
Sleeping
File size: 11,428 Bytes
daa4d56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 | """Owned, durable execution traces. Never records tokens, keys or SQL parameters."""
import contextvars
import functools
import hashlib
import inspect
import threading
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from sqlalchemy import event
_active = contextvars.ContextVar('workos_trace', default=None)
_parent = contextvars.ContextVar('workos_span', default=None)
_secret_names = {'authorization','password','access_token','refresh_token','api_key','key','headers','secret','token','user','u','self'}
def snapshot(value, depth=0):
if depth > 6:
return '[内容层级已折叠]'
if isinstance(value, bytes):
return {'bytes':len(value), 'sha256':hashlib.sha256(value).hexdigest()}
if isinstance(value, str):
return value[:16384] + ('…[截断]' if len(value)>16384 else '')
if value is None or isinstance(value,(int,float,bool)):
return value
if isinstance(value,dict):
return {str(k):snapshot(v,depth+1) for k,v in list(value.items())[:60] if str(k).lower() not in _secret_names}
if isinstance(value,(list,tuple)):
return [snapshot(v,depth+1) for v in value[:40]]
if hasattr(value,'document_id'):
return {k:getattr(value,k,None) for k in ('document_id','version','base_id','status','error_code','sha256')}
return {'type':type(value).__name__}
class Trace:
def __init__(self, Session, operation, *, trace_id=None, user=None, workspace=None, run_id=None):
self.Session=Session
self.id=trace_id or 'trace-'+uuid.uuid4().hex
self.operation=operation
self.user=user
self.workspace=workspace
self.run_id=run_id
self.started=time.perf_counter()
self.started_at=datetime.now(timezone.utc).isoformat()
self.spans=[]
self.database_groups={}
self.omitted_database_executions=0
self.chunks=[]
self.lock=threading.RLock()
self.closed=False
def __enter__(self):
self.token=_active.set(self)
self.parent_token=_parent.set(None)
return self
def start(self,module,operation,input_data=None,parent=None):
with self.lock:
if self.closed:return None
parent_id=parent if parent is not None else _parent.get()
group=None
if module=='Database':
group=(parent_id,operation,str(input_data))
if group in self.database_groups:
item=self.database_groups[group];item['executions']+=1
return item
if len(self.spans)>=350:
if module=='Database':
self.omitted_database_executions+=1;return None
victim=next((s for s in self.spans if s['module']=='Database' and s['status']=='ok'),None)
if victim is None:return None
self.omitted_database_executions+=victim.get('executions',1)
self.spans.remove(victim)
for k in [k for k,v in self.database_groups.items() if v is victim]:del self.database_groups[k]
item={'span_id':'s-'+uuid.uuid4().hex[:12], 'parent_id':parent_id,
'module':module,'operation':operation,'status':'running','started_ms':round((time.perf_counter()-self.started)*1000,2),
'input':snapshot(input_data),'result':None,'error':None}
self.spans.append(item)
if group is not None:
item['executions']=1;self.database_groups[group]=item
return item
def end(self,item,started,*,result=None,error=None):
if item is None:return
with self.lock:
elapsed=(time.perf_counter()-started)*1000
item['duration_ms']=round(elapsed+ (item.get('duration_ms',0) if item['module']=='Database' else 0),2)
item['status']='error' if error or item['status']=='error' else 'ok'
item['result']=snapshot(result)
if item['module']=='Database':item['result']={**(item['result'] or {}),'executions':item['executions'],'aggregation':'相同父阶段与SQL模板合并,参数未记录;耗时为各次执行之和。'}
if error:item['error']={'code':getattr(error,'code',type(error).__name__),'message':getattr(error,'message',type(error).__name__)}
def finish(self,status='ok'):
if self.closed:return
self.closed=True
if not self.user:return
if self.omitted_database_executions and self.spans:
self.spans[0]['trace_limits']={'omitted_database_executions':self.omitted_database_executions,'notice':'SQL 明细达到上限;优先保留业务模块和模型调用阶段。知识块账本独立分页保留。'}
from KnowledgeService.models import TraceRecord,TraceChunk
# Trace storage must not recursively trace itself.
token=_active.set(None)
try:
with self.Session.begin() as db:
db.add(TraceRecord(id=self.id,tenant=self.user['tenant_id'],owner=self.user['user_id'],workspace=self.workspace,
run_id=self.run_id,operation=self.operation,status=status,duration_ms=int((time.perf_counter()-self.started)*1000),
started_at=self.started_at,spans=self.spans))
for ordinal,chunk in enumerate(self.chunks,1):db.add(TraceChunk(trace_id=self.id,ordinal=ordinal,body=chunk))
finally:
_active.reset(token)
def __exit__(self,typ,exc,tb):
try:self.finish('error' if exc else 'ok')
finally:
_parent.reset(self.parent_token)
_active.reset(self.token)
def scope(user=None,workspace=None,run_id=None):
trace=_active.get()
if trace and not trace.closed:
if user:trace.user=user
if workspace:trace.workspace=workspace
if run_id:trace.run_id=run_id
def chunk_ledger(rows):
trace=_active.get()
if trace and not trace.closed:
with trace.lock:trace.chunks.extend(rows)
def trace_chunks(workspace,knowledge,user,wid,tid,after=0,limit=50):
from sqlalchemy import select
from KnowledgeService.models import TraceRecord,TraceChunk
workspace.assert_access(user,wid)
from APIGateway.app.errors import GatewayError
with workspace.Session() as db:
t=db.get(TraceRecord,tid)
if not t or (t.tenant,t.owner,t.workspace)!=(user['tenant_id'],user['user_id'],wid):raise GatewayError(404,'RESOURCE_NOT_FOUND','链路不存在或无权查看。')
rows=db.scalars(select(TraceChunk).where(TraceChunk.trace_id==tid,TraceChunk.ordinal>after).order_by(TraceChunk.ordinal).limit(limit+1)).all()
items=[]
for row in rows[:limit]:
try:knowledge.citation(user,row.body['citation_id'],wid)
except Exception as e:
if getattr(e,'status',None)==404:continue
raise
items.append({**row.body,'ledger_ordinal':row.ordinal})
return {'items':items,'next_cursor':rows[limit-1].ordinal if len(rows)>limit else None,'scope':'当前用户仍获授权的知识块'}
@contextmanager
def span(module,operation,input_data=None):
trace=_active.get()
started=time.perf_counter()
item=trace.start(module,operation,input_data) if trace else None
token=_parent.set(item['span_id']) if item else None
result={}
try:
yield result
except Exception as exc:
if trace:trace.end(item,started,error=exc)
raise
else:
if trace:trace.end(item,started,result=result.get('value',result))
finally:
if token:_parent.reset(token)
def traced(module,operation=None):
def decorate(fn):
signature=inspect.signature(fn)
def inputs(args,kwargs):
values=signature.bind(*args,**kwargs).arguments
# user identity comes only from verified Gateway dependency, never from payload.
user=values.get('user') or values.get('u')
wid=values.get('wid') or values.get('workspace_id')
scope(user, wid)
return {k:v for k,v in values.items() if k not in _secret_names}
if inspect.iscoroutinefunction(fn):
@functools.wraps(fn)
async def async_call(*args,**kwargs):
if not _active.get():return await fn(*args,**kwargs)
with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
value=await fn(*args,**kwargs)
output['value']=value
return value
return async_call
@functools.wraps(fn)
def call(*args,**kwargs):
if not _active.get():return fn(*args,**kwargs)
with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
value=fn(*args,**kwargs)
output['value']=value
return value
return call
return decorate
def instrument_engine(engine):
if getattr(engine,'_workos_traced',False):return
engine._workos_traced=True
@event.listens_for(engine,'before_cursor_execute')
def before(conn,cursor,statement,parameters,context,executemany):
trace=_active.get()
if trace and not trace.closed:
# SQL text uses bound parameters; parameter values are deliberately excluded.
sql=' '.join(statement.split())
item=trace.start('Database',sql.split(' ',1)[0],{'statement':sql[:1000],'parameters':'已隐藏','dialect':engine.dialect.name})
context._workos_span=(trace,item,time.perf_counter())
@event.listens_for(engine,'after_cursor_execute')
def after(conn,cursor,statement,parameters,context,executemany):
value=getattr(context,'_workos_span',None)
if value:
trace,item,started=value
trace.end(item,started,result={'row_count':cursor.rowcount,'executed':True})
@event.listens_for(engine,'handle_error')
def error(context):
value=getattr(context.execution_context,'_workos_span',None)
if value:
trace,item,started=value
trace.end(item,started,error=context.original_exception)
def trace_rows(workspace,user,wid):
from sqlalchemy import select,func
from KnowledgeService.models import TraceRecord,TraceChunk
workspace.assert_access(user,wid)
with workspace.Session() as db:
rows=db.scalars(select(TraceRecord).where(TraceRecord.tenant==user['tenant_id'],TraceRecord.owner==user['user_id'],
TraceRecord.workspace==wid).order_by(TraceRecord.started_at.desc()).limit(60)).all()
counts=dict(db.execute(select(TraceChunk.trace_id,func.count()).where(TraceChunk.trace_id.in_([r.id for r in rows])).group_by(TraceChunk.trace_id)).all())
return {'items':[{**{k:getattr(r,k) for k in ('id','run_id','operation','status','duration_ms','started_at','spans')},'chunk_count':counts.get(r.id,0)} for r in rows],
'limits':{'max_traces':60,'max_spans_per_trace':350,'max_text_chars':16384},
'notice':'记录真实模块调用与SQL执行;不记录鉴权令牌、模型密钥和SQL参数。长内容按上限截断。'}
|