Spaces:
Sleeping
Sleeping
Download observability.py from dellweb3/workos-office: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/observability.py
- Command line
-
hf download hf://spaces/dellweb3/workos-office/observability.py
-
curl -L -o observability.py https://huggingface.co/spaces/dellweb3/workos-office/resolve/main/observability.py
11.4 kB
| """Owned, durable execution traces. Never records tokens, keys or SQL parameters.""" | |
| import contextvars | |
| import functools | |
| import hashlib | |
| import inspect | |
| import threading | |
| import time | |
| import uuid | |
| from contextlib import contextmanager | |
| from datetime import datetime, timezone | |
| from sqlalchemy import event | |
| _active = contextvars.ContextVar('workos_trace', default=None) | |
| _parent = contextvars.ContextVar('workos_span', default=None) | |
| _secret_names = {'authorization','password','access_token','refresh_token','api_key','key','headers','secret','token','user','u','self'} | |
| def snapshot(value, depth=0): | |
| if depth > 6: | |
| return '[内容层级已折叠]' | |
| if isinstance(value, bytes): | |
| return {'bytes':len(value), 'sha256':hashlib.sha256(value).hexdigest()} | |
| if isinstance(value, str): | |
| return value[:16384] + ('…[截断]' if len(value)>16384 else '') | |
| if value is None or isinstance(value,(int,float,bool)): | |
| return value | |
| if isinstance(value,dict): | |
| return {str(k):snapshot(v,depth+1) for k,v in list(value.items())[:60] if str(k).lower() not in _secret_names} | |
| if isinstance(value,(list,tuple)): | |
| return [snapshot(v,depth+1) for v in value[:40]] | |
| if hasattr(value,'document_id'): | |
| return {k:getattr(value,k,None) for k in ('document_id','version','base_id','status','error_code','sha256')} | |
| return {'type':type(value).__name__} | |
| class Trace: | |
| def __init__(self, Session, operation, *, trace_id=None, user=None, workspace=None, run_id=None): | |
| self.Session=Session | |
| self.id=trace_id or 'trace-'+uuid.uuid4().hex | |
| self.operation=operation | |
| self.user=user | |
| self.workspace=workspace | |
| self.run_id=run_id | |
| self.started=time.perf_counter() | |
| self.started_at=datetime.now(timezone.utc).isoformat() | |
| self.spans=[] | |
| self.database_groups={} | |
| self.omitted_database_executions=0 | |
| self.chunks=[] | |
| self.lock=threading.RLock() | |
| self.closed=False | |
| def __enter__(self): | |
| self.token=_active.set(self) | |
| self.parent_token=_parent.set(None) | |
| return self | |
| def start(self,module,operation,input_data=None,parent=None): | |
| with self.lock: | |
| if self.closed:return None | |
| parent_id=parent if parent is not None else _parent.get() | |
| group=None | |
| if module=='Database': | |
| group=(parent_id,operation,str(input_data)) | |
| if group in self.database_groups: | |
| item=self.database_groups[group];item['executions']+=1 | |
| return item | |
| if len(self.spans)>=350: | |
| if module=='Database': | |
| self.omitted_database_executions+=1;return None | |
| victim=next((s for s in self.spans if s['module']=='Database' and s['status']=='ok'),None) | |
| if victim is None:return None | |
| self.omitted_database_executions+=victim.get('executions',1) | |
| self.spans.remove(victim) | |
| for k in [k for k,v in self.database_groups.items() if v is victim]:del self.database_groups[k] | |
| item={'span_id':'s-'+uuid.uuid4().hex[:12], 'parent_id':parent_id, | |
| 'module':module,'operation':operation,'status':'running','started_ms':round((time.perf_counter()-self.started)*1000,2), | |
| 'input':snapshot(input_data),'result':None,'error':None} | |
| self.spans.append(item) | |
| if group is not None: | |
| item['executions']=1;self.database_groups[group]=item | |
| return item | |
| def end(self,item,started,*,result=None,error=None): | |
| if item is None:return | |
| with self.lock: | |
| elapsed=(time.perf_counter()-started)*1000 | |
| item['duration_ms']=round(elapsed+ (item.get('duration_ms',0) if item['module']=='Database' else 0),2) | |
| item['status']='error' if error or item['status']=='error' else 'ok' | |
| item['result']=snapshot(result) | |
| if item['module']=='Database':item['result']={**(item['result'] or {}),'executions':item['executions'],'aggregation':'相同父阶段与SQL模板合并,参数未记录;耗时为各次执行之和。'} | |
| if error:item['error']={'code':getattr(error,'code',type(error).__name__),'message':getattr(error,'message',type(error).__name__)} | |
| def finish(self,status='ok'): | |
| if self.closed:return | |
| self.closed=True | |
| if not self.user:return | |
| if self.omitted_database_executions and self.spans: | |
| self.spans[0]['trace_limits']={'omitted_database_executions':self.omitted_database_executions,'notice':'SQL 明细达到上限;优先保留业务模块和模型调用阶段。知识块账本独立分页保留。'} | |
| from KnowledgeService.models import TraceRecord,TraceChunk | |
| # Trace storage must not recursively trace itself. | |
| token=_active.set(None) | |
| try: | |
| with self.Session.begin() as db: | |
| db.add(TraceRecord(id=self.id,tenant=self.user['tenant_id'],owner=self.user['user_id'],workspace=self.workspace, | |
| run_id=self.run_id,operation=self.operation,status=status,duration_ms=int((time.perf_counter()-self.started)*1000), | |
| started_at=self.started_at,spans=self.spans)) | |
| for ordinal,chunk in enumerate(self.chunks,1):db.add(TraceChunk(trace_id=self.id,ordinal=ordinal,body=chunk)) | |
| finally: | |
| _active.reset(token) | |
| def __exit__(self,typ,exc,tb): | |
| try:self.finish('error' if exc else 'ok') | |
| finally: | |
| _parent.reset(self.parent_token) | |
| _active.reset(self.token) | |
| def scope(user=None,workspace=None,run_id=None): | |
| trace=_active.get() | |
| if trace and not trace.closed: | |
| if user:trace.user=user | |
| if workspace:trace.workspace=workspace | |
| if run_id:trace.run_id=run_id | |
| def chunk_ledger(rows): | |
| trace=_active.get() | |
| if trace and not trace.closed: | |
| with trace.lock:trace.chunks.extend(rows) | |
| def trace_chunks(workspace,knowledge,user,wid,tid,after=0,limit=50): | |
| from sqlalchemy import select | |
| from KnowledgeService.models import TraceRecord,TraceChunk | |
| workspace.assert_access(user,wid) | |
| from APIGateway.app.errors import GatewayError | |
| with workspace.Session() as db: | |
| t=db.get(TraceRecord,tid) | |
| if not t or (t.tenant,t.owner,t.workspace)!=(user['tenant_id'],user['user_id'],wid):raise GatewayError(404,'RESOURCE_NOT_FOUND','链路不存在或无权查看。') | |
| rows=db.scalars(select(TraceChunk).where(TraceChunk.trace_id==tid,TraceChunk.ordinal>after).order_by(TraceChunk.ordinal).limit(limit+1)).all() | |
| items=[] | |
| for row in rows[:limit]: | |
| try:knowledge.citation(user,row.body['citation_id'],wid) | |
| except Exception as e: | |
| if getattr(e,'status',None)==404:continue | |
| raise | |
| items.append({**row.body,'ledger_ordinal':row.ordinal}) | |
| return {'items':items,'next_cursor':rows[limit-1].ordinal if len(rows)>limit else None,'scope':'当前用户仍获授权的知识块'} | |
| def span(module,operation,input_data=None): | |
| trace=_active.get() | |
| started=time.perf_counter() | |
| item=trace.start(module,operation,input_data) if trace else None | |
| token=_parent.set(item['span_id']) if item else None | |
| result={} | |
| try: | |
| yield result | |
| except Exception as exc: | |
| if trace:trace.end(item,started,error=exc) | |
| raise | |
| else: | |
| if trace:trace.end(item,started,result=result.get('value',result)) | |
| finally: | |
| if token:_parent.reset(token) | |
| def traced(module,operation=None): | |
| def decorate(fn): | |
| signature=inspect.signature(fn) | |
| def inputs(args,kwargs): | |
| values=signature.bind(*args,**kwargs).arguments | |
| # user identity comes only from verified Gateway dependency, never from payload. | |
| user=values.get('user') or values.get('u') | |
| wid=values.get('wid') or values.get('workspace_id') | |
| scope(user, wid) | |
| return {k:v for k,v in values.items() if k not in _secret_names} | |
| if inspect.iscoroutinefunction(fn): | |
| async def async_call(*args,**kwargs): | |
| if not _active.get():return await fn(*args,**kwargs) | |
| with span(module,operation or fn.__name__,inputs(args,kwargs)) as output: | |
| value=await fn(*args,**kwargs) | |
| output['value']=value | |
| return value | |
| return async_call | |
| def call(*args,**kwargs): | |
| if not _active.get():return fn(*args,**kwargs) | |
| with span(module,operation or fn.__name__,inputs(args,kwargs)) as output: | |
| value=fn(*args,**kwargs) | |
| output['value']=value | |
| return value | |
| return call | |
| return decorate | |
| def instrument_engine(engine): | |
| if getattr(engine,'_workos_traced',False):return | |
| engine._workos_traced=True | |
| def before(conn,cursor,statement,parameters,context,executemany): | |
| trace=_active.get() | |
| if trace and not trace.closed: | |
| # SQL text uses bound parameters; parameter values are deliberately excluded. | |
| sql=' '.join(statement.split()) | |
| item=trace.start('Database',sql.split(' ',1)[0],{'statement':sql[:1000],'parameters':'已隐藏','dialect':engine.dialect.name}) | |
| context._workos_span=(trace,item,time.perf_counter()) | |
| def after(conn,cursor,statement,parameters,context,executemany): | |
| value=getattr(context,'_workos_span',None) | |
| if value: | |
| trace,item,started=value | |
| trace.end(item,started,result={'row_count':cursor.rowcount,'executed':True}) | |
| def error(context): | |
| value=getattr(context.execution_context,'_workos_span',None) | |
| if value: | |
| trace,item,started=value | |
| trace.end(item,started,error=context.original_exception) | |
| def trace_rows(workspace,user,wid): | |
| from sqlalchemy import select,func | |
| from KnowledgeService.models import TraceRecord,TraceChunk | |
| workspace.assert_access(user,wid) | |
| with workspace.Session() as db: | |
| rows=db.scalars(select(TraceRecord).where(TraceRecord.tenant==user['tenant_id'],TraceRecord.owner==user['user_id'], | |
| TraceRecord.workspace==wid).order_by(TraceRecord.started_at.desc()).limit(60)).all() | |
| counts=dict(db.execute(select(TraceChunk.trace_id,func.count()).where(TraceChunk.trace_id.in_([r.id for r in rows])).group_by(TraceChunk.trace_id)).all()) | |
| return {'items':[{**{k:getattr(r,k) for k in ('id','run_id','operation','status','duration_ms','started_at','spans')},'chunk_count':counts.get(r.id,0)} for r in rows], | |
| 'limits':{'max_traces':60,'max_spans_per_trace':350,'max_text_chars':16384}, | |
| 'notice':'记录真实模块调用与SQL执行;不记录鉴权令牌、模型密钥和SQL参数。长内容按上限截断。'} | |