File size: 7,286 Bytes
064bfd6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 | // Centralized analytics/telemetry logging for tool permission decisions.
// All permission approve/reject events flow through logPermissionDecision(),
// which fans out to Statsig analytics, OTel telemetry, and code-edit metrics.
import { feature } from 'bun:bundle'
import {
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
logEvent,
} from 'src/services/analytics/index.js'
import { sanitizeToolNameForAnalytics } from 'src/services/analytics/metadata.js'
import { getCodeEditToolDecisionCounter } from '../../bootstrap/state.js'
import type { Tool as ToolType, ToolUseContext } from '../../Tool.js'
import { getLanguageName } from '../../utils/cliHighlight.js'
import { SandboxManager } from '../../utils/sandbox/sandbox-adapter.js'
import { logOTelEvent } from '../../utils/telemetry/events.js'
import type {
PermissionApprovalSource,
PermissionRejectionSource,
} from './PermissionContext.js'
type PermissionLogContext = {
tool: ToolType
input: unknown
toolUseContext: ToolUseContext
messageId: string
toolUseID: string
}
// Discriminated union: 'accept' pairs with approval sources, 'reject' with rejection sources
type PermissionDecisionArgs =
| { decision: 'accept'; source: PermissionApprovalSource | 'config' }
| { decision: 'reject'; source: PermissionRejectionSource | 'config' }
const CODE_EDITING_TOOLS = ['Edit', 'Write', 'NotebookEdit']
function isCodeEditingTool(toolName: string): boolean {
return CODE_EDITING_TOOLS.includes(toolName)
}
// Builds OTel counter attributes for code editing tools, enriching with
// language when the tool's target file path can be extracted from input
async function buildCodeEditToolAttributes(
tool: ToolType,
input: unknown,
decision: 'accept' | 'reject',
source: string,
): Promise<Record<string, string>> {
// Derive language from file path if the tool exposes one (e.g., Edit, Write)
let language: string | undefined
if (tool.getPath && input) {
const parseResult = tool.inputSchema.safeParse(input)
if (parseResult.success) {
const filePath = tool.getPath(parseResult.data)
if (filePath) {
language = await getLanguageName(filePath)
}
}
}
return {
decision,
source,
tool_name: tool.name,
...(language && { language }),
}
}
// Flattens structured source into a string label for analytics/OTel events
function sourceToString(
source: PermissionApprovalSource | PermissionRejectionSource,
): string {
if (
(feature('BASH_CLASSIFIER') || feature('TRANSCRIPT_CLASSIFIER')) &&
source.type === 'classifier'
) {
return 'classifier'
}
switch (source.type) {
case 'hook':
return 'hook'
case 'user':
return source.permanent ? 'user_permanent' : 'user_temporary'
case 'user_abort':
return 'user_abort'
case 'user_reject':
return 'user_reject'
default:
return 'unknown'
}
}
function baseMetadata(
messageId: string,
toolName: string,
waitMs: number | undefined,
): { [key: string]: boolean | number | undefined } {
return {
messageID:
messageId as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
toolName: sanitizeToolNameForAnalytics(toolName),
sandboxEnabled: SandboxManager.isSandboxingEnabled(),
// Only include wait time when the user was actually prompted (not auto-approved)
...(waitMs !== undefined && { waiting_for_user_permission_ms: waitMs }),
}
}
// Emits a distinct analytics event name per approval source for funnel analysis
function logApprovalEvent(
tool: ToolType,
messageId: string,
source: PermissionApprovalSource | 'config',
waitMs: number | undefined,
): void {
if (source === 'config') {
// Auto-approved by allowlist in settings -- no user wait time
logEvent(
'tengu_tool_use_granted_in_config',
baseMetadata(messageId, tool.name, undefined),
)
return
}
if (
(feature('BASH_CLASSIFIER') || feature('TRANSCRIPT_CLASSIFIER')) &&
source.type === 'classifier'
) {
logEvent(
'tengu_tool_use_granted_by_classifier',
baseMetadata(messageId, tool.name, waitMs),
)
return
}
switch (source.type) {
case 'user':
logEvent(
source.permanent
? 'tengu_tool_use_granted_in_prompt_permanent'
: 'tengu_tool_use_granted_in_prompt_temporary',
baseMetadata(messageId, tool.name, waitMs),
)
break
case 'hook':
logEvent('tengu_tool_use_granted_by_permission_hook', {
...baseMetadata(messageId, tool.name, waitMs),
permanent: source.permanent ?? false,
})
break
default:
break
}
}
// Rejections share a single event name, differentiated by metadata fields
function logRejectionEvent(
tool: ToolType,
messageId: string,
source: PermissionRejectionSource | 'config',
waitMs: number | undefined,
): void {
if (source === 'config') {
// Denied by denylist in settings
logEvent(
'tengu_tool_use_denied_in_config',
baseMetadata(messageId, tool.name, undefined),
)
return
}
logEvent('tengu_tool_use_rejected_in_prompt', {
...baseMetadata(messageId, tool.name, waitMs),
// Distinguish hook rejections from user rejections via separate fields
...(source.type === 'hook'
? { isHook: true }
: {
hasFeedback:
source.type === 'user_reject' ? source.hasFeedback : false,
}),
})
}
// Single entry point for all permission decision logging. Called by permission
// handlers after every approve/reject. Fans out to: analytics events, OTel
// telemetry, code-edit OTel counters, and toolUseContext decision storage.
function logPermissionDecision(
ctx: PermissionLogContext,
args: PermissionDecisionArgs,
permissionPromptStartTimeMs?: number,
): void {
const { tool, input, toolUseContext, messageId, toolUseID } = ctx
const { decision, source } = args
const waiting_for_user_permission_ms =
permissionPromptStartTimeMs !== undefined
? Date.now() - permissionPromptStartTimeMs
: undefined
// Log the analytics event
if (args.decision === 'accept') {
logApprovalEvent(
tool,
messageId,
args.source,
waiting_for_user_permission_ms,
)
} else {
logRejectionEvent(
tool,
messageId,
args.source,
waiting_for_user_permission_ms,
)
}
const sourceString = source === 'config' ? 'config' : sourceToString(source)
// Track code editing tool metrics
if (isCodeEditingTool(tool.name)) {
void buildCodeEditToolAttributes(tool, input, decision, sourceString).then(
attributes => getCodeEditToolDecisionCounter()?.add(1, attributes),
)
}
// Persist decision on the context so downstream code can inspect what happened
if (!toolUseContext.toolDecisions) {
toolUseContext.toolDecisions = new Map()
}
toolUseContext.toolDecisions.set(toolUseID, {
source: sourceString,
decision,
timestamp: Date.now(),
})
void logOTelEvent('tool_decision', {
decision,
source: sourceString,
tool_name: sanitizeToolNameForAnalytics(tool.name),
})
}
export { isCodeEditingTool, buildCodeEditToolAttributes, logPermissionDecision }
export type { PermissionLogContext, PermissionDecisionArgs }
|