workos-office / observability.py
dellweb3's picture
Work OS validated demo local-v0.8 (part 2)
daa4d56 verified
Raw History Blame Contribute Delete
11.4 kB
"""Owned, durable execution traces. Never records tokens, keys or SQL parameters."""
import contextvars
import functools
import hashlib
import inspect
import threading
import time
import uuid
from contextlib import contextmanager
from datetime import datetime, timezone
from sqlalchemy import event
_active = contextvars.ContextVar('workos_trace', default=None)
_parent = contextvars.ContextVar('workos_span', default=None)
_secret_names = {'authorization','password','access_token','refresh_token','api_key','key','headers','secret','token','user','u','self'}
def snapshot(value, depth=0):
if depth > 6:
return '[内容层级已折叠]'
if isinstance(value, bytes):
return {'bytes':len(value), 'sha256':hashlib.sha256(value).hexdigest()}
if isinstance(value, str):
return value[:16384] + ('…[截断]' if len(value)>16384 else '')
if value is None or isinstance(value,(int,float,bool)):
return value
if isinstance(value,dict):
return {str(k):snapshot(v,depth+1) for k,v in list(value.items())[:60] if str(k).lower() not in _secret_names}
if isinstance(value,(list,tuple)):
return [snapshot(v,depth+1) for v in value[:40]]
if hasattr(value,'document_id'):
return {k:getattr(value,k,None) for k in ('document_id','version','base_id','status','error_code','sha256')}
return {'type':type(value).__name__}
class Trace:
def __init__(self, Session, operation, *, trace_id=None, user=None, workspace=None, run_id=None):
self.Session=Session
self.id=trace_id or 'trace-'+uuid.uuid4().hex
self.operation=operation
self.user=user
self.workspace=workspace
self.run_id=run_id
self.started=time.perf_counter()
self.started_at=datetime.now(timezone.utc).isoformat()
self.spans=[]
self.database_groups={}
self.omitted_database_executions=0
self.chunks=[]
self.lock=threading.RLock()
self.closed=False
def __enter__(self):
self.token=_active.set(self)
self.parent_token=_parent.set(None)
return self
def start(self,module,operation,input_data=None,parent=None):
with self.lock:
if self.closed:return None
parent_id=parent if parent is not None else _parent.get()
group=None
if module=='Database':
group=(parent_id,operation,str(input_data))
if group in self.database_groups:
item=self.database_groups[group];item['executions']+=1
return item
if len(self.spans)>=350:
if module=='Database':
self.omitted_database_executions+=1;return None
victim=next((s for s in self.spans if s['module']=='Database' and s['status']=='ok'),None)
if victim is None:return None
self.omitted_database_executions+=victim.get('executions',1)
self.spans.remove(victim)
for k in [k for k,v in self.database_groups.items() if v is victim]:del self.database_groups[k]
item={'span_id':'s-'+uuid.uuid4().hex[:12], 'parent_id':parent_id,
'module':module,'operation':operation,'status':'running','started_ms':round((time.perf_counter()-self.started)*1000,2),
'input':snapshot(input_data),'result':None,'error':None}
self.spans.append(item)
if group is not None:
item['executions']=1;self.database_groups[group]=item
return item
def end(self,item,started,*,result=None,error=None):
if item is None:return
with self.lock:
elapsed=(time.perf_counter()-started)*1000
item['duration_ms']=round(elapsed+ (item.get('duration_ms',0) if item['module']=='Database' else 0),2)
item['status']='error' if error or item['status']=='error' else 'ok'
item['result']=snapshot(result)
if item['module']=='Database':item['result']={**(item['result'] or {}),'executions':item['executions'],'aggregation':'相同父阶段与SQL模板合并,参数未记录;耗时为各次执行之和。'}
if error:item['error']={'code':getattr(error,'code',type(error).__name__),'message':getattr(error,'message',type(error).__name__)}
def finish(self,status='ok'):
if self.closed:return
self.closed=True
if not self.user:return
if self.omitted_database_executions and self.spans:
self.spans[0]['trace_limits']={'omitted_database_executions':self.omitted_database_executions,'notice':'SQL 明细达到上限;优先保留业务模块和模型调用阶段。知识块账本独立分页保留。'}
from KnowledgeService.models import TraceRecord,TraceChunk
# Trace storage must not recursively trace itself.
token=_active.set(None)
try:
with self.Session.begin() as db:
db.add(TraceRecord(id=self.id,tenant=self.user['tenant_id'],owner=self.user['user_id'],workspace=self.workspace,
run_id=self.run_id,operation=self.operation,status=status,duration_ms=int((time.perf_counter()-self.started)*1000),
started_at=self.started_at,spans=self.spans))
for ordinal,chunk in enumerate(self.chunks,1):db.add(TraceChunk(trace_id=self.id,ordinal=ordinal,body=chunk))
finally:
_active.reset(token)
def __exit__(self,typ,exc,tb):
try:self.finish('error' if exc else 'ok')
finally:
_parent.reset(self.parent_token)
_active.reset(self.token)
def scope(user=None,workspace=None,run_id=None):
trace=_active.get()
if trace and not trace.closed:
if user:trace.user=user
if workspace:trace.workspace=workspace
if run_id:trace.run_id=run_id
def chunk_ledger(rows):
trace=_active.get()
if trace and not trace.closed:
with trace.lock:trace.chunks.extend(rows)
def trace_chunks(workspace,knowledge,user,wid,tid,after=0,limit=50):
from sqlalchemy import select
from KnowledgeService.models import TraceRecord,TraceChunk
workspace.assert_access(user,wid)
from APIGateway.app.errors import GatewayError
with workspace.Session() as db:
t=db.get(TraceRecord,tid)
if not t or (t.tenant,t.owner,t.workspace)!=(user['tenant_id'],user['user_id'],wid):raise GatewayError(404,'RESOURCE_NOT_FOUND','链路不存在或无权查看。')
rows=db.scalars(select(TraceChunk).where(TraceChunk.trace_id==tid,TraceChunk.ordinal>after).order_by(TraceChunk.ordinal).limit(limit+1)).all()
items=[]
for row in rows[:limit]:
try:knowledge.citation(user,row.body['citation_id'],wid)
except Exception as e:
if getattr(e,'status',None)==404:continue
raise
items.append({**row.body,'ledger_ordinal':row.ordinal})
return {'items':items,'next_cursor':rows[limit-1].ordinal if len(rows)>limit else None,'scope':'当前用户仍获授权的知识块'}
@contextmanager
def span(module,operation,input_data=None):
trace=_active.get()
started=time.perf_counter()
item=trace.start(module,operation,input_data) if trace else None
token=_parent.set(item['span_id']) if item else None
result={}
try:
yield result
except Exception as exc:
if trace:trace.end(item,started,error=exc)
raise
else:
if trace:trace.end(item,started,result=result.get('value',result))
finally:
if token:_parent.reset(token)
def traced(module,operation=None):
def decorate(fn):
signature=inspect.signature(fn)
def inputs(args,kwargs):
values=signature.bind(*args,**kwargs).arguments
# user identity comes only from verified Gateway dependency, never from payload.
user=values.get('user') or values.get('u')
wid=values.get('wid') or values.get('workspace_id')
scope(user, wid)
return {k:v for k,v in values.items() if k not in _secret_names}
if inspect.iscoroutinefunction(fn):
@functools.wraps(fn)
async def async_call(*args,**kwargs):
if not _active.get():return await fn(*args,**kwargs)
with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
value=await fn(*args,**kwargs)
output['value']=value
return value
return async_call
@functools.wraps(fn)
def call(*args,**kwargs):
if not _active.get():return fn(*args,**kwargs)
with span(module,operation or fn.__name__,inputs(args,kwargs)) as output:
value=fn(*args,**kwargs)
output['value']=value
return value
return call
return decorate
def instrument_engine(engine):
if getattr(engine,'_workos_traced',False):return
engine._workos_traced=True
@event.listens_for(engine,'before_cursor_execute')
def before(conn,cursor,statement,parameters,context,executemany):
trace=_active.get()
if trace and not trace.closed:
# SQL text uses bound parameters; parameter values are deliberately excluded.
sql=' '.join(statement.split())
item=trace.start('Database',sql.split(' ',1)[0],{'statement':sql[:1000],'parameters':'已隐藏','dialect':engine.dialect.name})
context._workos_span=(trace,item,time.perf_counter())
@event.listens_for(engine,'after_cursor_execute')
def after(conn,cursor,statement,parameters,context,executemany):
value=getattr(context,'_workos_span',None)
if value:
trace,item,started=value
trace.end(item,started,result={'row_count':cursor.rowcount,'executed':True})
@event.listens_for(engine,'handle_error')
def error(context):
value=getattr(context.execution_context,'_workos_span',None)
if value:
trace,item,started=value
trace.end(item,started,error=context.original_exception)
def trace_rows(workspace,user,wid):
from sqlalchemy import select,func
from KnowledgeService.models import TraceRecord,TraceChunk
workspace.assert_access(user,wid)
with workspace.Session() as db:
rows=db.scalars(select(TraceRecord).where(TraceRecord.tenant==user['tenant_id'],TraceRecord.owner==user['user_id'],
TraceRecord.workspace==wid).order_by(TraceRecord.started_at.desc()).limit(60)).all()
counts=dict(db.execute(select(TraceChunk.trace_id,func.count()).where(TraceChunk.trace_id.in_([r.id for r in rows])).group_by(TraceChunk.trace_id)).all())
return {'items':[{**{k:getattr(r,k) for k in ('id','run_id','operation','status','duration_ms','started_at','spans')},'chunk_count':counts.get(r.id,0)} for r in rows],
'limits':{'max_traces':60,'max_spans_per_trace':350,'max_text_chars':16384},
'notice':'记录真实模块调用与SQL执行;不记录鉴权令牌、模型密钥和SQL参数。长内容按上限截断。'}