Spaces:
Sleeping
Sleeping
Download apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 4.83 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
-
curl -L -o sendToSearchIndex.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/sendToSearchIndex.ts
4.83 kB
| /** | |
| * Transformer: Send Document to Search Index | |
| * | |
| * Integrates with the existing scraper transformer stack. | |
| * Queues documents for real-time search indexing. | |
| * | |
| * Sampling: Controlled via SEARCH_INDEX_SAMPLE_RATE (0.0-1.0) | |
| * - 0.1 = 10% of documents indexed (recommended for initial rollout) | |
| * - 1.0 = 100% of documents indexed (full production) | |
| */ | |
| import { Document } from "../../../controllers/v1/types"; | |
| import { config } from "../../../config"; | |
| import { indexDocumentIfEnabled } from "../../../lib/search-index-client"; | |
| import { logger as _logger } from "../../../lib/logger"; | |
| import { Meta } from ".."; | |
| /** | |
| * Check if document should be indexed for search | |
| */ | |
| function shouldIndexForSearch(meta: Meta, document: Document): boolean { | |
| if (meta.internalOptions.isParse) { | |
| return false; | |
| } | |
| if (meta.internalOptions.zeroDataRetention) { | |
| return false; | |
| } | |
| // Lockdown must not forward the target URL to any external service. | |
| if (meta.options.lockdown) { | |
| return false; | |
| } | |
| const statusCode = document.metadata.statusCode; | |
| if (statusCode < 200 || statusCode >= 300) { | |
| return false; | |
| } | |
| // Check if markdown content exists and is substantial | |
| const markdown = document.markdown ?? ""; | |
| if (markdown.length < 200) { | |
| return false; | |
| } | |
| // Don't index if has auth headers (private content) | |
| if ( | |
| meta.options.headers && | |
| (meta.options.headers["Authorization"] || meta.options.headers["Cookie"]) | |
| ) { | |
| return false; | |
| } | |
| // Don't index PDFs without parsing | |
| const isPdf = document.metadata.contentType?.includes("pdf"); | |
| if (isPdf && !document.markdown) { | |
| return false; | |
| } | |
| return true; | |
| } | |
| /** | |
| * Determine if this document should be sampled for indexing | |
| */ | |
| function shouldSampleDocument(): boolean { | |
| // Get sample rate from environment (default 10% for safe rollout) | |
| const sampleRate = config.SEARCH_INDEX_SAMPLE_RATE; | |
| // Validate sample rate | |
| if (isNaN(sampleRate) || sampleRate < 0 || sampleRate > 1) { | |
| _logger.warn("Invalid SEARCH_INDEX_SAMPLE_RATE, using 0.1 (10%)", { | |
| value: sampleRate, | |
| }); | |
| return Math.random() < 0.1; | |
| } | |
| // Sample based on rate | |
| return Math.random() < sampleRate; | |
| } | |
| /** | |
| * Transformer: Send document to search index | |
| */ | |
| export async function sendDocumentToSearchIndex( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| // Check if search indexing is enabled via the SEARCH_SERVICE_URL | |
| const searchIndexEnabled = | |
| config.ENABLE_SEARCH_INDEX && config.SEARCH_SERVICE_URL; | |
| meta.logger.debug("Sending document to search index", { | |
| url: meta.url, | |
| searchIndexEnabled, | |
| }); | |
| if (!searchIndexEnabled) { | |
| return document; | |
| } | |
| // Apply sampling (canary rollout) | |
| if (!shouldSampleDocument()) { | |
| meta.logger.debug("Document not sampled for search indexing", { | |
| url: meta.url, | |
| sampleRate: config.SEARCH_INDEX_SAMPLE_RATE, | |
| }); | |
| return document; | |
| } | |
| // Check if document should be indexed | |
| if (!shouldIndexForSearch(meta, document)) { | |
| meta.logger.debug("Document not suitable for search index", { | |
| url: meta.url, | |
| statusCode: document.metadata.statusCode, | |
| markdownLength: document.markdown?.length ?? 0, | |
| }); | |
| return document; | |
| } | |
| // Get the indexId from document metadata (set by sendDocumentToIndex transformer) | |
| // Format as GCS path: {indexId}.json | |
| const gcsPath = document.metadata.indexId; | |
| // Remove indexId from metadata after extracting it (internal field, shouldn't be exposed to user) | |
| delete document.metadata.indexId; | |
| // Send to search service via HTTP (async, don't block scraper) | |
| (async () => { | |
| try { | |
| await indexDocumentIfEnabled( | |
| { | |
| url: meta.url, | |
| resolvedUrl: | |
| document.metadata.url ?? | |
| document.metadata.sourceURL ?? | |
| meta.rewrittenUrl ?? | |
| meta.url, | |
| title: | |
| document.metadata.title ?? document.metadata.ogTitle ?? undefined, | |
| description: | |
| document.metadata.description ?? | |
| document.metadata.ogDescription ?? | |
| undefined, | |
| markdown: document.markdown ?? "", | |
| html: document.rawHtml ?? "", | |
| statusCode: document.metadata.statusCode, | |
| gcsPath: gcsPath, | |
| screenshotUrl: document.screenshot ?? undefined, | |
| language: document.metadata.language ?? "en", | |
| country: meta.options.location?.country ?? undefined, | |
| isMobile: meta.options.mobile ?? false, | |
| }, | |
| meta.logger, | |
| ); | |
| meta.logger.debug("Sent document to search service", { | |
| url: meta.url, | |
| }); | |
| } catch (error) { | |
| meta.logger.error("Failed to send document to search service", { | |
| error: (error as Error).message, | |
| url: meta.url, | |
| }); | |
| } | |
| })(); | |
| return document; | |
| } | |