Spaces:
Sleeping
Sleeping
Download apps/api/src/services/sentry.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 5.46 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/services/sentry.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/services/sentry.ts
-
curl -L -o sentry.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/services/sentry.ts
5.46 kB
| import * as Sentry from "@sentry/node"; | |
| import { logger } from "../lib/logger"; | |
| import { config } from "../config"; | |
| import { | |
| AddFeatureError, | |
| RemoveFeatureError, | |
| EngineError, | |
| } from "../scraper/scrapeURL/error"; | |
| import { AbortManagerThrownError } from "../scraper/scrapeURL/lib/abortManager"; | |
| import { JobCancelledError } from "../lib/error"; | |
| import { isQueueFullError } from "../lib/queue-full-error"; | |
| type CaptureContext = { | |
| tags?: Record<string, string>; | |
| extra?: Record<string, any>; | |
| level?: Sentry.SeverityLevel; | |
| fingerprint?: string[]; | |
| contexts?: Record<string, any>; | |
| user?: { | |
| id?: string; | |
| username?: string; | |
| email?: string; | |
| ip_address?: string; | |
| [key: string]: any; | |
| }; | |
| [key: string]: any; | |
| }; | |
| const transportableErrorCodes = [ | |
| "SCRAPE_ALL_ENGINES_FAILED", | |
| "SCRAPE_DNS_RESOLUTION_ERROR", | |
| "SCRAPE_SITE_ERROR", | |
| "SCRAPE_SSL_ERROR", | |
| "SCRAPE_PROXY_SELECTION_ERROR", | |
| "SCRAPE_ZDR_VIOLATION_ERROR", | |
| "SCRAPE_UNSUPPORTED_FILE_ERROR", | |
| "SCRAPE_PDF_ANTIBOT_ERROR", | |
| "SCRAPE_ACTION_ERROR", | |
| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR", | |
| "SCRAPE_PDF_PREFETCH_FAILED", | |
| "SCRAPE_DOCUMENT_ANTIBOT_ERROR", | |
| "SCRAPE_DOCUMENT_PREFETCH_FAILED", | |
| "SCRAPE_TIMEOUT", | |
| "MAP_TIMEOUT", | |
| "SCRAPE_UNKNOWN_ERROR", | |
| "SCRAPE_RACED_REDIRECT_ERROR", | |
| "SCRAPE_SITEMAP_ERROR", | |
| "CRAWL_DENIAL", | |
| ]; | |
| function getStringField( | |
| error: Record<string, unknown>, | |
| fields: string[], | |
| ): string { | |
| for (const field of fields) { | |
| if (error[field] !== undefined && error[field] !== null) { | |
| return String(error[field]); | |
| } | |
| } | |
| return ""; | |
| } | |
| function getErrorMessage(error: unknown): string { | |
| if (error instanceof Error) { | |
| return String(error.message || ""); | |
| } | |
| if (typeof error === "string") { | |
| return error; | |
| } | |
| if (error && typeof error === "object") { | |
| return getStringField(error as Record<string, unknown>, [ | |
| "message", | |
| "value", | |
| ]); | |
| } | |
| return ""; | |
| } | |
| export function shouldIgnoreSentryException(error: unknown): boolean { | |
| if (isQueueFullError(error)) { | |
| return true; | |
| } | |
| if (!error || typeof error !== "object") { | |
| return false; | |
| } | |
| if ( | |
| error instanceof AddFeatureError || | |
| error instanceof RemoveFeatureError || | |
| error instanceof AbortManagerThrownError || | |
| error instanceof EngineError || | |
| error instanceof JobCancelledError | |
| ) { | |
| return true; | |
| } | |
| // We sometimes rethrow TransportableErrors across process boundaries as a | |
| // plain Error with message like `${code}|${json}` (see error-serde.ts). | |
| // In that case, `error.code` is missing, so extract it from the message. | |
| const errorCodeFromField = "code" in error ? String(error.code) : ""; | |
| const errorMessage = getErrorMessage(error); | |
| // Ignore cancellation errors (fallback for serialized errors) | |
| if ( | |
| errorMessage === "Parent crawl/batch scrape was cancelled" || | |
| errorMessage.includes("Parent crawl/batch scrape was cancelled") | |
| ) { | |
| return true; | |
| } | |
| const errorCodeFromMessage = | |
| errorCodeFromField || | |
| (errorMessage.includes("|") ? errorMessage.split("|", 1)[0] : ""); | |
| const errorCode = errorCodeFromMessage; | |
| return transportableErrorCodes.includes(errorCode); | |
| } | |
| if (config.SENTRY_DSN) { | |
| logger.info("Setting up Sentry..."); | |
| Sentry.init({ | |
| dsn: config.SENTRY_DSN, | |
| integrations: integrations => [ | |
| ...integrations, | |
| Sentry.vercelAIIntegration({ | |
| recordInputs: false, | |
| recordOutputs: false, | |
| }), | |
| ], | |
| tracesSampler: samplingContext => { | |
| // trace all AI spans, sample 1% of all others | |
| return samplingContext.name?.startsWith("ai.") | |
| ? 1.0 | |
| : config.SENTRY_TRACE_SAMPLE_RATE; | |
| }, | |
| sampleRate: config.SENTRY_ERROR_SAMPLE_RATE, | |
| serverName: config.NUQ_POD_NAME, | |
| environment: config.SENTRY_ENVIRONMENT, | |
| beforeSend(event, hint) { | |
| const zeroDataRetention = | |
| event.tags?.zeroDataRetention === "true" || | |
| event.tags?.zero_data_retention === "true" || | |
| event.extra?.zeroDataRetention === true || | |
| event.extra?.zero_data_retention === true; | |
| if (zeroDataRetention) { | |
| return null; | |
| } | |
| const error = hint?.originalException; | |
| const sentryException = event.exception?.values?.[0]; | |
| if ( | |
| shouldIgnoreSentryException(error) || | |
| shouldIgnoreSentryException(sentryException) | |
| ) { | |
| return null; | |
| } | |
| return event; | |
| }, | |
| }); | |
| } | |
| export function captureExceptionWithZdrCheck( | |
| error: any, | |
| context?: (CaptureContext & { zeroDataRetention?: boolean }) | null, | |
| ) { | |
| const zeroDataRetention = | |
| context?.zeroDataRetention ?? | |
| (context?.extra as any)?.zeroDataRetention ?? | |
| (context?.data as any)?.zeroDataRetention; | |
| if (zeroDataRetention) { | |
| return; | |
| } | |
| if (shouldIgnoreSentryException(error)) { | |
| return; | |
| } | |
| const { zeroDataRetention: _zdr, ...sentryContext } = context || {}; | |
| return Sentry.captureException(error, sentryContext); | |
| } | |
| export function applyZdrScope(zeroDataRetention?: boolean) { | |
| if (!zeroDataRetention) { | |
| return; | |
| } | |
| const scope = Sentry.getCurrentScope(); | |
| scope.setTag("zeroDataRetention", "true"); | |
| scope.setExtra("zeroDataRetention", true); | |
| } | |
| /** | |
| * Set the service type tag for this Sentry instance | |
| * This helps distinguish between API server and worker errors in Sentry | |
| */ | |
| export function setSentryServiceTag(serviceType: string) { | |
| if (config.SENTRY_DSN) { | |
| Sentry.setTag("service_type", serviceType); | |
| } | |
| } | |