Spaces:
Sleeping
Sleeping
Download apps/api/src/scraper/scrapeURL/transformers/agent.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 2.37 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/agent.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/scraper/scrapeURL/transformers/agent.ts
-
curl -L -o agent.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/transformers/agent.ts
2.37 kB
| import { Document } from "../../../controllers/v1/types"; | |
| import { Meta } from ".."; | |
| import { logger } from "../../../lib/logger"; | |
| import { parseMarkdown } from "../../../lib/html-to-markdown"; | |
| import { smartScrape, SmartScrapeResult } from "../lib/smartScrape"; | |
| import { hasFormatOfType } from "../../../lib/format-utils"; | |
| export async function performAgent( | |
| meta: Meta, | |
| document: Document, | |
| ): Promise<Document> { | |
| if (meta.internalOptions.v1Agent?.prompt) { | |
| if (meta.internalOptions.zeroDataRetention) { | |
| document.warning = | |
| "Agent is not supported with zero data retention." + | |
| (document.warning ? " " + document.warning : ""); | |
| return document; | |
| } | |
| const url: string | undefined = document.url || document.metadata.sourceURL; | |
| if (!url) { | |
| logger.error( | |
| "document.url or document.metadata.sourceURL is undefined -- this is unexpected", | |
| ); | |
| // throw new Error("document.url or document.metadata.sourceURL is undefined -- this is unexpected"); | |
| return document; | |
| } | |
| const prompt = meta.internalOptions.v1Agent?.prompt ?? undefined; | |
| const sessionId = meta.internalOptions.v1Agent?.sessionId ?? undefined; | |
| let smartscrapeResults: SmartScrapeResult; | |
| try { | |
| smartscrapeResults = await smartScrape({ | |
| url, | |
| prompt, | |
| sessionId, | |
| scrapeId: meta.id, | |
| costTracking: meta.costTracking, | |
| }); | |
| } catch (error) { | |
| if (error instanceof Error && error.message === "Cost limit exceeded") { | |
| logger.error("Cost limit exceeded", { error }); | |
| document.warning = | |
| "Smart scrape cost limit exceeded." + | |
| (document.warning ? " " + document.warning : ""); | |
| return document; | |
| } else { | |
| throw error; | |
| } | |
| } | |
| const html = | |
| smartscrapeResults.scrapedPages[ | |
| smartscrapeResults.scrapedPages.length - 1 | |
| ].html; | |
| if (hasFormatOfType(meta.options.formats, "markdown")) { | |
| const requestId = meta.id || meta.internalOptions.crawlId; | |
| const markdown = await parseMarkdown(html, { | |
| logger: meta.logger, | |
| requestId, | |
| zeroDataRetention: meta.internalOptions.zeroDataRetention, | |
| }); | |
| document.markdown = markdown; | |
| } | |
| if (hasFormatOfType(meta.options.formats, "html")) { | |
| document.html = html; | |
| } | |
| } | |
| return document; | |
| } | |