Spaces:
Sleeping
Sleeping
Download apps/api/src/services/webhook/delivery.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 8.06 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/services/webhook/delivery.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/services/webhook/delivery.ts
-
curl -L -o delivery.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/services/webhook/delivery.ts
8.06 kB
| import undici from "undici"; | |
| import { config } from "../../config"; | |
| import { createHmac } from "crypto"; | |
| import { logger as _logger, logger } from "../../lib/logger"; | |
| import { | |
| getSecureDispatcherNoCookies, | |
| isIPPrivate, | |
| } from "../../scraper/scrapeURL/engines/utils/safeFetch"; | |
| import type { | |
| WebhookConfig, | |
| WebhookEvent, | |
| WebhookEventDataMap, | |
| WebhookQueueMessage, | |
| } from "./types"; | |
| import { redisEvictConnection } from "../redis"; | |
| import { supabase_service } from "../supabase"; | |
| import { webhookQueue } from "./queue"; | |
| import { randomUUID } from "crypto"; | |
| const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue"; | |
| const WEBHOOK_INSERT_BATCH_SIZE = 1000; | |
| type WebhookSendResult = { | |
| attempted: boolean; | |
| delivered?: boolean; | |
| queued?: boolean; | |
| skipped?: boolean; | |
| statusCode?: number; | |
| }; | |
| export function webhookEventMatchesFilter( | |
| configuredEvents: string[] | undefined, | |
| event: WebhookEvent, | |
| ): boolean { | |
| if (!configuredEvents?.length) { | |
| return true; | |
| } | |
| const legacySubType = event.split(".")[1]; | |
| const namespaceSuffix = event.split(".").slice(1).join("."); | |
| return ( | |
| configuredEvents.includes(event) || | |
| configuredEvents.includes(legacySubType) || | |
| configuredEvents.includes(namespaceSuffix) | |
| ); | |
| } | |
| export class WebhookSender { | |
| private config: WebhookConfig; | |
| private secret?: string; | |
| private context: { teamId: string; jobId: string; v0: boolean }; | |
| private logger: any; | |
| constructor( | |
| config: WebhookConfig, | |
| secret: string | undefined, | |
| context: { teamId: string; jobId: string; v0: boolean }, | |
| ) { | |
| this.config = config; | |
| this.secret = secret; | |
| this.context = context; | |
| this.logger = _logger.child({ | |
| module: "webhook-sender", | |
| teamId: context.teamId, | |
| jobId: context.jobId, | |
| isV0: context.v0, | |
| }); | |
| } | |
| async send<T extends WebhookEvent>( | |
| event: T, | |
| data: WebhookEventDataMap[T], | |
| ): Promise<WebhookSendResult> { | |
| if (!this.shouldSendEvent(event)) { | |
| return { attempted: false, skipped: true }; | |
| } | |
| const payload = { | |
| success: data.success, | |
| type: event, | |
| [this.context.v0 ? "jobId" : "id"]: this.context.jobId, | |
| webhookId: randomUUID(), // Unique ID for this webhook delivery (used for e.g. retries) | |
| data: "data" in data ? data.data : [], | |
| error: "error" in data ? data.error : undefined, | |
| metadata: this.config.metadata || undefined, | |
| }; | |
| const delivery = this.deliver( | |
| payload, | |
| (data as any)?.scrapeId ?? undefined, | |
| ); | |
| if (data.awaitWebhook) { | |
| return { attempted: true, ...(await delivery) }; | |
| } | |
| delivery.catch(() => {}); | |
| return { | |
| attempted: true, | |
| delivered: false, | |
| queued: this.usesWebhookQueue(), | |
| }; | |
| } | |
| private shouldSendEvent(event: WebhookEvent): boolean { | |
| if (config.DISABLE_WEBHOOK_DELIVERY) { | |
| return false; | |
| } | |
| return webhookEventMatchesFilter(this.config.events, event); | |
| } | |
| private usesWebhookQueue(): boolean { | |
| return Boolean(config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL); | |
| } | |
| private async deliver( | |
| payload: any, | |
| scrapeId?: string, | |
| ): Promise<Omit<WebhookSendResult, "attempted">> { | |
| const webhookHost = new URL(this.config.url).hostname; | |
| if (isIPPrivate(webhookHost) && config.ALLOW_LOCAL_WEBHOOKS !== true) { | |
| this.logger.warn("Aborting webhook call to private IP address", { | |
| webhookUrl: this.config.url, | |
| }); | |
| return { delivered: false, skipped: true }; | |
| } | |
| if (this.usesWebhookQueue()) { | |
| const queueMessage: WebhookQueueMessage = { | |
| webhook_url: this.config.url, | |
| payload, | |
| headers: this.config.headers, | |
| team_id: this.context.teamId, | |
| job_id: this.context.jobId, | |
| scrape_id: scrapeId ?? null, | |
| event: payload.type, | |
| timeout_ms: this.context.v0 ? 30000 : 10000, | |
| }; | |
| try { | |
| await webhookQueue.publish(queueMessage); | |
| this.logger.info("Webhook queued for delivery", { | |
| webhookUrl: this.config.url, | |
| event: payload.type, | |
| }); | |
| } catch (error) { | |
| this.logger.error("Failed to queue webhook", { | |
| error, | |
| webhookUrl: this.config.url, | |
| }); | |
| throw error; | |
| } | |
| return { delivered: false, queued: true }; | |
| } | |
| const payloadString = JSON.stringify(payload); | |
| const headers: Record<string, string> = { | |
| "Content-Type": "application/json", | |
| ...this.config.headers, | |
| }; | |
| if (this.secret) { | |
| const hmac = createHmac("sha256", this.secret); | |
| hmac.update(payloadString); | |
| headers["X-Firecrawl-Signature"] = `sha256=${hmac.digest("hex")}`; | |
| } | |
| const abortController = new AbortController(); | |
| const timeoutHandle = setTimeout( | |
| () => { | |
| if (abortController) { | |
| abortController.abort(); | |
| } | |
| }, | |
| this.context.v0 ? 30000 : 10000, | |
| ); | |
| try { | |
| const res = await undici.fetch(this.config.url, { | |
| method: "POST", | |
| headers, | |
| body: payloadString, | |
| dispatcher: getSecureDispatcherNoCookies(), | |
| signal: abortController.signal, | |
| }); | |
| if (!res.ok) { | |
| throw new Error(`Unexpected response status: ${res.status}`); | |
| } | |
| await logWebhook({ | |
| success: res.status >= 200 && res.status < 300, | |
| teamId: this.context.teamId, | |
| crawlId: this.context.jobId, // this is legacy naming, we should rename it to jobId at some point | |
| scrapeId, | |
| url: this.config.url, | |
| event: payload.type, | |
| statusCode: res.status, | |
| }); | |
| return { delivered: true, queued: false, statusCode: res.status }; | |
| } catch (error) { | |
| this.logger.error("Failed to send webhook", { | |
| error, | |
| webhookUrl: this.config.url, | |
| }); | |
| await logWebhook({ | |
| success: false, | |
| teamId: this.context.teamId, | |
| crawlId: this.context.jobId, // same as above | |
| scrapeId, | |
| url: this.config.url, | |
| event: payload.type, | |
| error: | |
| error instanceof Error | |
| ? error.message | |
| : typeof error === "string" | |
| ? error | |
| : undefined, | |
| statusCode: | |
| typeof (error as any)?.status === "number" | |
| ? (error as any).status | |
| : undefined, | |
| }); | |
| throw error; | |
| } finally { | |
| if (timeoutHandle) clearTimeout(timeoutHandle); | |
| } | |
| } | |
| get metadata(): Record<string, string> { | |
| return this.config.metadata || {}; | |
| } | |
| } | |
| export async function getWebhookInsertQueueLength(): Promise<number> { | |
| return (await redisEvictConnection.llen(WEBHOOK_INSERT_QUEUE_KEY)) ?? 0; | |
| } | |
| export async function processWebhookInsertJobs() { | |
| const jobs = | |
| (await redisEvictConnection.lpop( | |
| WEBHOOK_INSERT_QUEUE_KEY, | |
| WEBHOOK_INSERT_BATCH_SIZE, | |
| )) ?? []; | |
| if (jobs.length === 0) return; | |
| const parsedJobs = jobs.map(x => JSON.parse(x)); | |
| _logger.info("Webhook inserter found jobs to insert", { | |
| jobCount: parsedJobs.length, | |
| }); | |
| try { | |
| await supabase_service.from("webhook_logs").insert(parsedJobs); | |
| _logger.info("Webhook inserter inserted jobs", { | |
| jobCount: parsedJobs.length, | |
| }); | |
| } catch (error) { | |
| _logger.error("Webhook inserter failed to insert jobs", { | |
| error, | |
| jobCount: parsedJobs.length, | |
| }); | |
| } | |
| } | |
| async function logWebhook(data: { | |
| success: boolean; | |
| error?: string; | |
| teamId: string; | |
| crawlId: string; | |
| scrapeId?: string; | |
| url: string; | |
| statusCode?: number; | |
| event: WebhookEvent; | |
| }): Promise<void> { | |
| try { | |
| await redisEvictConnection.rpush( | |
| WEBHOOK_INSERT_QUEUE_KEY, | |
| JSON.stringify({ | |
| success: data.success, | |
| error: data.error ?? null, | |
| team_id: data.teamId, | |
| crawl_id: data.crawlId, | |
| scrape_id: data.scrapeId ?? null, | |
| url: data.url, | |
| status_code: data.statusCode ?? null, | |
| event: data.event, | |
| }), | |
| ); | |
| } catch (error) { | |
| logger.error("Error logging webhook", { | |
| error, | |
| teamId: data.teamId, | |
| }); | |
| } | |
| } | |