Spaces:
Sleeping
Sleeping
Download apps/api/src/scraper/scrapeURL/transformers/index.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 19.1 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/index.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/scraper/scrapeURL/transformers/index.ts
-
curl -L -o index.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/index.ts
19.1 kB
| import { parseMarkdown } from "../../../lib/html-to-markdown"; | |
| import { Meta } from ".."; | |
| import { Document } from "../../../controllers/v2/types"; | |
| import { htmlTransform } from "../lib/removeUnwantedElements"; | |
| import { extractLinks } from "../lib/extractLinks"; | |
| import { extractImages } from "../lib/extractImages"; | |
| import { extractMetadata } from "../lib/extractMetadata"; | |
| import { | |
| performLLMExtract, | |
| performSummary, | |
| performCleanContent, | |
| } from "./llmExtract"; | |
| import { performQuery } from "./query"; | |
| import { uploadScreenshot } from "./uploadScreenshot"; | |
| import { removeBase64Images } from "./removeBase64Images"; | |
| import { performAgent } from "./agent"; | |
| import { performAttributes } from "./performAttributes"; | |
| import { deriveDiff } from "./diff"; | |
| import { fetchAudio } from "./audio"; | |
| import { fetchVideo } from "./video"; | |
| import { useIndex, useSearchIndex } from "../../../services/index"; | |
| import { sendDocumentToIndex } from "../engines/index/index"; | |
| import { sendDocumentToSearchIndex } from "./sendToSearchIndex"; | |
| import { hasFormatOfType } from "../../../lib/format-utils"; | |
| import { brandingTransformer } from "../../../lib/branding/transformer"; | |
| import { indexerQueue } from "../../../services/indexing/indexer-queue"; | |
| import { config } from "../../../config"; | |
| type Transformer = ( | |
| meta: Meta, | |
| document: Document, | |
| ) => Document | Promise<Document>; | |
| async function deriveMetadataFromRawHTML( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| if (document.rawHtml === undefined) { | |
| throw new Error( | |
| "rawHtml is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| document.metadata = { | |
| ...(await extractMetadata(meta, document.rawHtml)), | |
| ...document.metadata, | |
| }; | |
| return document; | |
| } | |
| async function deriveHTMLFromRawHTML( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| if (document.rawHtml === undefined) { | |
| throw new Error( | |
| "rawHtml is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| document.html = await htmlTransform( | |
| document.rawHtml, | |
| document.metadata.url ?? | |
| document.metadata.sourceURL ?? | |
| meta.rewrittenUrl ?? | |
| meta.url, | |
| meta.options, | |
| ); | |
| return document; | |
| } | |
| async function deriveMarkdownFromHTML( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| if (document.html === undefined) { | |
| throw new Error( | |
| "html is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| // Only derive markdown if markdown format is requested or if formats that require markdown are requested: | |
| // - changeTracking requires markdown | |
| // - json format requires markdown (for LLM extraction) | |
| // - summary format requires markdown (for summarization) | |
| // - question/highlights/query formats require markdown (for page-level answers) | |
| const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown"); | |
| const hasChangeTracking = hasFormatOfType( | |
| meta.options.formats, | |
| "changeTracking", | |
| ); | |
| const hasJson = hasFormatOfType(meta.options.formats, "json"); | |
| const hasSummary = hasFormatOfType(meta.options.formats, "summary"); | |
| const hasQuestion = hasFormatOfType(meta.options.formats, "question"); | |
| const hasHighlights = hasFormatOfType(meta.options.formats, "highlights"); | |
| const hasQuery = hasFormatOfType(meta.options.formats, "query"); | |
| if ( | |
| !hasMarkdown && | |
| !hasChangeTracking && | |
| !hasJson && | |
| !hasSummary && | |
| !hasQuestion && | |
| !hasHighlights && | |
| !hasQuery && | |
| !meta.options.onlyCleanContent | |
| ) { | |
| return document; | |
| } | |
| // Skip markdown derivation if a postprocessor already set it | |
| if (document.metadata.postprocessorsUsed?.length && document.markdown) { | |
| meta.logger.debug( | |
| "Skipping markdown derivation - postprocessor already set markdown", | |
| { postprocessorsUsed: document.metadata.postprocessorsUsed }, | |
| ); | |
| return document; | |
| } | |
| if (document.metadata.contentType?.includes("application/json")) { | |
| if (document.rawHtml === undefined) { | |
| throw new Error( | |
| "rawHtml is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| document.markdown = "```json\n" + document.rawHtml + "\n```"; | |
| return document; | |
| } | |
| // Use scrape ID or crawl ID as request_id for tracing | |
| const requestId = meta.id || meta.internalOptions.crawlId; | |
| document.markdown = await parseMarkdown(document.html, { | |
| logger: meta.logger, | |
| requestId, | |
| zeroDataRetention: meta.internalOptions.zeroDataRetention, | |
| }); | |
| if ( | |
| meta.options.onlyMainContent === true && | |
| (!document.markdown || document.markdown.trim().length === 0) | |
| ) { | |
| meta.logger.info( | |
| "Main content extraction resulted in empty markdown, falling back to full content extraction", | |
| ); | |
| const fallbackMeta = { | |
| ...meta, | |
| options: { | |
| ...meta.options, | |
| onlyMainContent: false, | |
| }, | |
| }; | |
| document = await deriveHTMLFromRawHTML(fallbackMeta, document); | |
| document.markdown = await parseMarkdown(document.html, { | |
| logger: meta.logger, | |
| requestId, | |
| zeroDataRetention: meta.internalOptions.zeroDataRetention, | |
| }); | |
| meta.logger.info("Fallback to full content extraction completed", { | |
| markdownLength: document.markdown?.length || 0, | |
| }); | |
| } | |
| return document; | |
| } | |
| async function deriveLinksFromHTML( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| if (document.html === undefined) { | |
| throw new Error( | |
| "html is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| const rate = config.INDEXER_TRAFFIC_SHARE | |
| ? Math.max(0, Math.min(1, Number(config.INDEXER_TRAFFIC_SHARE))) | |
| : 0; | |
| const shouldForwardTraffic = | |
| rate > 0 && Math.random() <= rate && !!config.INDEXER_RABBITMQ_URL; | |
| const forwardToIndexer = | |
| !meta.internalOptions.isParse && | |
| !!meta.internalOptions.teamId && | |
| !meta.internalOptions.teamId?.includes("robots-txt") && | |
| !meta.internalOptions.teamId?.includes("sitemap") && | |
| shouldForwardTraffic; | |
| const requiresLinks = !!hasFormatOfType(meta.options.formats, "links"); | |
| if (!forwardToIndexer && !requiresLinks) { | |
| return document; | |
| } | |
| document.links = await extractLinks( | |
| document.html, | |
| document.metadata.url ?? | |
| document.metadata.sourceURL ?? | |
| meta.rewrittenUrl ?? | |
| meta.url, | |
| ); | |
| if (forwardToIndexer) { | |
| try { | |
| let linksDeduped: Set<string> = new Set(); | |
| if (!!document.links) { | |
| linksDeduped = new Set([...document.links]); | |
| } | |
| indexerQueue | |
| .sendToWorker({ | |
| id: meta.id, | |
| type: "links", | |
| discovery_url: | |
| document.metadata.url ?? | |
| document.metadata.sourceURL ?? | |
| meta.rewrittenUrl ?? | |
| meta.url, | |
| urls: [...linksDeduped], | |
| }) | |
| .catch(error => { | |
| meta.logger.error("Failed to queue links for indexing", { | |
| error: (error as Error)?.message, | |
| url: meta.url, | |
| }); | |
| }); | |
| } catch (error) { | |
| meta.logger.error("Failed to queue links for indexing", { | |
| error: (error as Error)?.message, | |
| url: meta.url, | |
| }); | |
| } | |
| } | |
| if (!requiresLinks) { | |
| delete document.links; | |
| } | |
| return document; | |
| } | |
| async function deriveImagesFromHTML( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| // Only derive if the formats has images | |
| if (hasFormatOfType(meta.options.formats, "images")) { | |
| if (document.html === undefined) { | |
| throw new Error( | |
| "html is undefined -- this transformer is being called out of order", | |
| ); | |
| } | |
| document.images = await extractImages( | |
| document.html, | |
| document.metadata.url ?? | |
| document.metadata.sourceURL ?? | |
| meta.rewrittenUrl ?? | |
| meta.url, | |
| ); | |
| } | |
| return document; | |
| } | |
| async function deriveBrandingFromActions( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| const hasBranding = hasFormatOfType(meta.options.formats, "branding"); | |
| if (!hasBranding) { | |
| return document; | |
| } | |
| if (document.branding !== undefined) { | |
| return document; | |
| } | |
| /** | |
| * Find the branding return in the actions javascript returns | |
| * @see src/scraper/scrapeURL/engines/fire-engine/scripts/branding.js | |
| */ | |
| const brandingReturnIndex = document.actions?.javascriptReturns?.findIndex( | |
| x => x.type === "object" && "branding" in (x.value as any), | |
| ); | |
| if (brandingReturnIndex === -1 || brandingReturnIndex === undefined) { | |
| return document; | |
| } | |
| // cast as any since this is js return, we might need to validate this | |
| const javascriptReturn = document.actions!.javascriptReturns![ | |
| brandingReturnIndex | |
| ].value as any; | |
| const rawBranding = javascriptReturn?.branding; | |
| document.actions!.javascriptReturns!.splice(brandingReturnIndex, 1); | |
| document.branding = await brandingTransformer(meta, document, rawBranding); | |
| return document; | |
| } | |
| function coerceFieldsToFormats(meta: Meta, document: Document): Document { | |
| const hasMarkdown = hasFormatOfType(meta.options.formats, "markdown"); | |
| const hasRawHtml = hasFormatOfType(meta.options.formats, "rawHtml"); | |
| const hasHtml = hasFormatOfType(meta.options.formats, "html"); | |
| const hasLinks = hasFormatOfType(meta.options.formats, "links"); | |
| const hasImages = hasFormatOfType(meta.options.formats, "images"); | |
| const hasChangeTracking = hasFormatOfType( | |
| meta.options.formats, | |
| "changeTracking", | |
| ); | |
| const hasJson = hasFormatOfType(meta.options.formats, "json"); | |
| const hasScreenshot = hasFormatOfType(meta.options.formats, "screenshot"); | |
| const hasSummary = hasFormatOfType(meta.options.formats, "summary"); | |
| const hasBranding = hasFormatOfType(meta.options.formats, "branding"); | |
| const hasQuestionFormat = hasFormatOfType(meta.options.formats, "question"); | |
| const hasHighlightsFormat = hasFormatOfType( | |
| meta.options.formats, | |
| "highlights", | |
| ); | |
| const hasLegacyQueryFormat = hasFormatOfType(meta.options.formats, "query"); | |
| const hasAnswerFormat = hasQuestionFormat || hasLegacyQueryFormat; | |
| if (!hasMarkdown && document.markdown !== undefined) { | |
| delete document.markdown; | |
| } else if (hasMarkdown && document.markdown === undefined) { | |
| meta.logger.warn( | |
| "Request had format: markdown, but there was no markdown field in the result.", | |
| ); | |
| } | |
| if (!hasRawHtml && document.rawHtml !== undefined) { | |
| delete document.rawHtml; | |
| } else if (hasRawHtml && document.rawHtml === undefined) { | |
| meta.logger.warn( | |
| "Request had format: rawHtml, but there was no rawHtml field in the result.", | |
| ); | |
| } | |
| if (!hasHtml && document.html !== undefined) { | |
| delete document.html; | |
| } else if (hasHtml && document.html === undefined) { | |
| meta.logger.warn( | |
| "Request had format: html, but there was no html field in the result.", | |
| ); | |
| } | |
| if (!hasScreenshot && document.screenshot !== undefined) { | |
| meta.logger.warn( | |
| "Removed screenshot from Document because it wasn't in formats -- this is very wasteful and indicates a bug.", | |
| ); | |
| delete document.screenshot; | |
| } else if (hasScreenshot && document.screenshot === undefined) { | |
| meta.logger.warn( | |
| "Request had format: screenshot / screenshot@fullPage, but there was no screenshot field in the result.", | |
| ); | |
| } | |
| if (!hasLinks && document.links !== undefined) { | |
| meta.logger.warn( | |
| "Removed links from Document because it wasn't in formats -- this is wasteful and indicates a bug.", | |
| ); | |
| delete document.links; | |
| } else if (hasLinks && document.links === undefined) { | |
| meta.logger.warn( | |
| "Request had format: links, but there was no links field in the result.", | |
| ); | |
| } | |
| if (!hasImages && document.images !== undefined) { | |
| meta.logger.warn( | |
| "Removed images from Document because it wasn't in formats -- this is wasteful and indicates a bug.", | |
| { hasImages, hasImagesField: document.images !== undefined }, | |
| ); | |
| delete document.images; | |
| } else if (hasImages && document.images === undefined) { | |
| meta.logger.warn( | |
| "Request had format: images, but there was no images field in the result.", | |
| { hasImages, hasImagesField: document.images !== undefined }, | |
| ); | |
| } | |
| // Handle v1 backward compatibility - don't delete fields based on v1OriginalFormat | |
| const shouldKeepExtract = meta.internalOptions.v1OriginalFormat === "extract"; | |
| const shouldKeepJson = meta.internalOptions.v1OriginalFormat === "json"; | |
| // Debug logging for v1 format investigation | |
| if (meta.internalOptions.v1OriginalFormat) { | |
| meta.logger.debug("coerceFieldsToFormats v1 format debug", { | |
| v1OriginalFormat: meta.internalOptions.v1OriginalFormat, | |
| hasJson: !!hasJson, | |
| shouldKeepExtract, | |
| shouldKeepJson, | |
| hasExtractField: document.extract !== undefined, | |
| hasJsonField: document.json !== undefined, | |
| }); | |
| } | |
| if ( | |
| !hasJson && | |
| (document.extract !== undefined || document.json !== undefined) | |
| ) { | |
| // For v1 API, keep the field specified by v1OriginalFormat | |
| if (!shouldKeepExtract && document.extract !== undefined) { | |
| meta.logger.warn( | |
| "Removed extract from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.", | |
| ); | |
| delete document.extract; | |
| } | |
| if (!shouldKeepJson && document.json !== undefined) { | |
| meta.logger.warn( | |
| "Removed json from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.", | |
| ); | |
| delete document.json; | |
| } | |
| } else if ( | |
| hasJson && | |
| document.extract === undefined && | |
| document.json === undefined | |
| ) { | |
| meta.logger.warn( | |
| "Request had format json, but there was no json field in the result.", | |
| ); | |
| } | |
| if (!hasSummary && document.summary !== undefined) { | |
| meta.logger.warn( | |
| "Removed summary from Document because it wasn't in formats -- this is wasteful and indicates a bug.", | |
| ); | |
| delete document.summary; | |
| } else if (hasSummary && document.summary === undefined) { | |
| meta.logger.warn( | |
| "Request had format summary, but there was no summary field in the result.", | |
| ); | |
| } | |
| if (!hasAnswerFormat && document.answer !== undefined) { | |
| meta.logger.warn( | |
| "Removed answer from Document because question/query wasn't in formats -- this is wasteful and indicates a bug.", | |
| ); | |
| delete document.answer; | |
| } else if (hasAnswerFormat && document.answer === undefined) { | |
| meta.logger.warn( | |
| "Request had format question/query, but there was no answer field in the result.", | |
| ); | |
| } | |
| if (!hasHighlightsFormat && document.highlights !== undefined) { | |
| meta.logger.warn( | |
| "Removed highlights from Document because highlights wasn't in formats -- this is wasteful and indicates a bug.", | |
| ); | |
| delete document.highlights; | |
| } else if (hasHighlightsFormat && document.highlights === undefined) { | |
| meta.logger.warn( | |
| "Request had format highlights, but there was no highlights field in the result.", | |
| ); | |
| } | |
| if (!hasBranding && document.branding !== undefined) { | |
| meta.logger.warn( | |
| "Removed branding from Document because it wasn't in formats -- this indicates the engine returned unexpected data.", | |
| ); | |
| delete document.branding; | |
| } else if (hasBranding && document.branding === undefined) { | |
| meta.logger.warn( | |
| "Request had format branding, but there was no branding field in the result.", | |
| ); | |
| } | |
| const hasAudio = hasFormatOfType(meta.options.formats, "audio"); | |
| if (!hasAudio && document.audio !== undefined) { | |
| delete document.audio; | |
| } else if (hasAudio && document.audio === undefined) { | |
| meta.logger.warn( | |
| "Request had format: audio, but there was no audio field in the result.", | |
| ); | |
| } | |
| const hasVideo = hasFormatOfType(meta.options.formats, "video"); | |
| if (!hasVideo && document.video !== undefined) { | |
| delete document.video; | |
| } else if (hasVideo && document.video === undefined) { | |
| meta.logger.warn( | |
| "Request had format: video, but there was no video field in the result.", | |
| ); | |
| } | |
| if (!hasChangeTracking && document.changeTracking !== undefined) { | |
| meta.logger.warn( | |
| "Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.", | |
| ); | |
| delete document.changeTracking; | |
| } else if (hasChangeTracking && document.changeTracking === undefined) { | |
| meta.logger.warn( | |
| "Request had format changeTracking, but there was no changeTracking field in the result.", | |
| ); | |
| } | |
| if ( | |
| document.changeTracking && | |
| !hasChangeTracking?.modes?.includes("git-diff") && | |
| document.changeTracking.diff !== undefined | |
| ) { | |
| meta.logger.warn( | |
| "Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.", | |
| ); | |
| delete document.changeTracking.diff; | |
| } | |
| if ( | |
| document.changeTracking && | |
| !hasChangeTracking?.modes?.includes("json") && | |
| document.changeTracking.json !== undefined | |
| ) { | |
| meta.logger.warn( | |
| "Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.", | |
| ); | |
| delete document.changeTracking.json; | |
| } | |
| if (meta.options.actions === undefined || meta.options.actions.length === 0) { | |
| delete document.actions; | |
| } else if (document.actions) { | |
| // Check if all action arrays are empty | |
| const hasScreenshots = | |
| document.actions.screenshots && document.actions.screenshots.length > 0; | |
| const hasScrapes = | |
| document.actions.scrapes && document.actions.scrapes.length > 0; | |
| const hasJsReturns = | |
| document.actions.javascriptReturns && | |
| document.actions.javascriptReturns.length > 0; | |
| const hasPdfs = document.actions.pdfs && document.actions.pdfs.length > 0; | |
| if (!hasScreenshots && !hasScrapes && !hasJsReturns && !hasPdfs) { | |
| delete document.actions; | |
| } | |
| } | |
| return document; | |
| } | |
| // TODO: allow some of these to run in parallel | |
| const transformerStack: Transformer[] = [ | |
| deriveHTMLFromRawHTML, | |
| deriveMarkdownFromHTML, | |
| performCleanContent, | |
| deriveLinksFromHTML, | |
| deriveImagesFromHTML, | |
| deriveBrandingFromActions, | |
| deriveMetadataFromRawHTML, | |
| uploadScreenshot, | |
| ...(useIndex ? [sendDocumentToIndex] : []), | |
| ...(useSearchIndex ? [sendDocumentToSearchIndex] : []), // Add to search index for real-time search | |
| performLLMExtract, | |
| performSummary, | |
| performQuery, | |
| performAttributes, | |
| performAgent, | |
| removeBase64Images, | |
| deriveDiff, | |
| fetchAudio, | |
| fetchVideo, | |
| coerceFieldsToFormats, | |
| ]; | |
| export async function executeTransformers( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| const executions: [string, number][] = []; | |
| for (const transformer of transformerStack) { | |
| const _meta = { | |
| ...meta, | |
| logger: meta.logger.child({ | |
| method: "executeTransformers/" + transformer.name, | |
| }), | |
| }; | |
| const start = Date.now(); | |
| document = await transformer(_meta, document); | |
| executions.push([transformer.name, Date.now() - start]); | |
| } | |
| meta.logger.debug("Executed transformers.", { executions }); | |
| return document; | |
| } | |