Spaces:
Sleeping
Sleeping
Download apps/api/src/controllers/v0/crawl.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/controllers/v0/crawl.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/controllers/v0/crawl.ts
-
curl -L -o crawl.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/controllers/v0/crawl.ts
10 kB
| import { Request, Response } from "express"; | |
| import { checkTeamCredits } from "../../../src/services/billing/credit_billing"; | |
| import { authenticateUser } from "../auth"; | |
| import { RateLimiterMode } from "../../../src/types"; | |
| import { addScrapeJob } from "../../../src/services/queue-jobs"; | |
| import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist"; | |
| import { validateIdempotencyKey } from "../../../src/services/idempotency/validate"; | |
| import { createIdempotencyKey } from "../../../src/services/idempotency/create"; | |
| import { | |
| defaultCrawlPageOptions, | |
| defaultCrawlerOptions, | |
| defaultOrigin, | |
| } from "../../../src/lib/default-values"; | |
| import { v7 as uuidv7 } from "uuid"; | |
| import { logger } from "../../../src/lib/logger"; | |
| import { | |
| addCrawlJob, | |
| addCrawlJobs, | |
| crawlToCrawler, | |
| finishCrawlKickoff, | |
| lockURL, | |
| lockURLs, | |
| markCrawlActive, | |
| saveCrawl, | |
| StoredCrawl, | |
| } from "../../../src/lib/crawl-redis"; | |
| import { redisEvictConnection } from "../../../src/services/redis"; | |
| import { checkAndUpdateURL } from "../../../src/lib/validateUrl"; | |
| import * as Sentry from "@sentry/node"; | |
| import { getJobPriority } from "../../lib/job-priority"; | |
| import { url as urlSchema } from "../v1/types"; | |
| import { ZodError } from "zod"; | |
| import { UNSUPPORTED_SITE_MESSAGE } from "../../lib/strings"; | |
| import { fromV0ScrapeOptions } from "../v2/types"; | |
| import { isSelfHosted } from "../../lib/deployment"; | |
| import { crawlGroup } from "../../services/worker/nuq"; | |
| import { logRequest } from "../../services/logging/log_job"; | |
| import { getScrapeZDR } from "../../lib/zdr-helpers"; | |
| export async function crawlController(req: Request, res: Response) { | |
| try { | |
| const auth = await authenticateUser(req, res, RateLimiterMode.Crawl); | |
| if (!auth.success) { | |
| return res.status(auth.status).json({ error: auth.error }); | |
| } | |
| const { team_id, chunk } = auth; | |
| if (getScrapeZDR(chunk?.flags) === "forced") { | |
| return res.status(400).json({ | |
| error: | |
| "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API.", | |
| }); | |
| } | |
| const id = uuidv7(); | |
| await logRequest({ | |
| id, | |
| kind: "crawl", | |
| api_version: "v0", | |
| team_id, | |
| origin: req.body.origin ?? "api", | |
| integration: req.body.integration, | |
| target_hint: req.body.url ?? "", | |
| zeroDataRetention: false, // not supported on v0 | |
| api_key_id: chunk?.api_key_id ?? null, | |
| }); | |
| redisEvictConnection.sadd("teams_using_v0", team_id).catch(error => | |
| logger.error("Failed to add team to teams_using_v0", { | |
| error, | |
| team_id, | |
| }), | |
| ); | |
| redisEvictConnection | |
| .sadd("teams_using_v0:" + team_id, "crawl:" + id) | |
| .catch(error => | |
| logger.error("Failed to add team to teams_using_v0 (2)", { | |
| error, | |
| team_id, | |
| }), | |
| ); | |
| if (req.headers["x-idempotency-key"]) { | |
| const isIdempotencyValid = await validateIdempotencyKey(req); | |
| if (!isIdempotencyValid) { | |
| return res.status(409).json({ error: "Idempotency key already used" }); | |
| } | |
| try { | |
| createIdempotencyKey(req); | |
| } catch (error) { | |
| logger.error(error); | |
| return res.status(500).json({ error: error.message }); | |
| } | |
| } | |
| const crawlerOptions = { | |
| ...defaultCrawlerOptions, | |
| ...req.body.crawlerOptions, | |
| }; | |
| const pageOptions = { ...defaultCrawlPageOptions, ...req.body.pageOptions }; | |
| if (Array.isArray(crawlerOptions.includes)) { | |
| for (const x of crawlerOptions.includes) { | |
| try { | |
| new RegExp(x); | |
| } catch (e) { | |
| return res.status(400).json({ error: e.message }); | |
| } | |
| } | |
| } | |
| if (Array.isArray(crawlerOptions.excludes)) { | |
| for (const x of crawlerOptions.excludes) { | |
| try { | |
| new RegExp(x); | |
| } catch (e) { | |
| return res.status(400).json({ error: e.message }); | |
| } | |
| } | |
| } | |
| const limitCheck = req.body?.crawlerOptions?.limit ?? 1; | |
| const { | |
| success: creditsCheckSuccess, | |
| message: creditsCheckMessage, | |
| remainingCredits, | |
| } = await checkTeamCredits(chunk, team_id, limitCheck); | |
| if (!creditsCheckSuccess) { | |
| return res.status(402).json({ | |
| error: isSelfHosted() | |
| ? "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. Please check your server configuration." | |
| : "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at help@firecrawl.com", | |
| }); | |
| } | |
| // TODO: need to do this to v1 | |
| crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit); | |
| let url = urlSchema.parse(req.body.url); | |
| if (!url) { | |
| return res.status(400).json({ error: "Url is required" }); | |
| } | |
| if (typeof url !== "string") { | |
| return res.status(400).json({ error: "URL must be a string" }); | |
| } | |
| try { | |
| url = checkAndUpdateURL(url).url; | |
| } catch (e) { | |
| return res | |
| .status(e instanceof Error && e.message === "Invalid URL" ? 400 : 500) | |
| .json({ error: e.message ?? e }); | |
| } | |
| if ( | |
| isUrlBlocked(url, auth.chunk?.flags ?? null, { | |
| team_id: auth.chunk?.team_id ?? team_id, | |
| origin: req.body?.origin ?? null, | |
| }) | |
| ) { | |
| return res.status(403).json({ | |
| error: UNSUPPORTED_SITE_MESSAGE, | |
| }); | |
| } | |
| // if (mode === "single_urls" && !url.includes(",")) { // NOTE: do we need this? | |
| // try { | |
| // const a = new WebScraperDataProvider(); | |
| // await a.setOptions({ | |
| // jobId: uuidv7(), | |
| // mode: "single_urls", | |
| // urls: [url], | |
| // crawlerOptions: { ...crawlerOptions, returnOnlyUrls: true }, | |
| // pageOptions: pageOptions, | |
| // }); | |
| // const docs = await a.getDocuments(false, (progress) => { | |
| // job.updateProgress({ | |
| // current: progress.current, | |
| // total: progress.total, | |
| // current_step: "SCRAPING", | |
| // current_url: progress.currentDocumentUrl, | |
| // }); | |
| // }); | |
| // return res.json({ | |
| // success: true, | |
| // documents: docs, | |
| // }); | |
| // } catch (error) { | |
| // logger.error(error); | |
| // return res.status(500).json({ error: error.message }); | |
| // } | |
| // } | |
| const { scrapeOptions, internalOptions } = fromV0ScrapeOptions( | |
| pageOptions, | |
| undefined, | |
| undefined, | |
| team_id, | |
| ); | |
| internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter | |
| internalOptions.saveScrapeResultToGCS = process.env | |
| .GCS_FIRE_ENGINE_BUCKET_NAME | |
| ? true | |
| : false; | |
| delete (scrapeOptions as any).timeout; | |
| const sc: StoredCrawl = { | |
| originUrl: url, | |
| crawlerOptions, | |
| scrapeOptions, | |
| internalOptions, | |
| team_id, | |
| createdAt: Date.now(), | |
| }; | |
| const crawler = crawlToCrawler(id, sc, auth.chunk?.flags ?? null); | |
| try { | |
| sc.robots = await crawler.getRobotsTxt(); | |
| } catch (_) {} | |
| await crawlGroup.addGroup( | |
| id, | |
| sc.team_id, | |
| (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000, | |
| ); | |
| await saveCrawl(id, sc); | |
| await markCrawlActive(id); | |
| await finishCrawlKickoff(id); | |
| const sitemap = sc.crawlerOptions.ignoreSitemap | |
| ? 0 | |
| : await crawler.tryGetSitemap(async urls => { | |
| if (urls.length === 0) return; | |
| let jobPriority = await getJobPriority({ | |
| team_id, | |
| basePriority: 21, | |
| }); | |
| const billing = { endpoint: "crawl" as const, jobId: id }; | |
| const jobs = urls.map(url => { | |
| const uuid = uuidv7(); | |
| return { | |
| jobId: uuid, | |
| data: { | |
| url, | |
| mode: "single_urls" as const, | |
| crawlerOptions, | |
| scrapeOptions, | |
| internalOptions, | |
| team_id, | |
| origin: req.body.origin ?? defaultOrigin, | |
| integration: req.body.integration, | |
| billing, | |
| crawl_id: id, | |
| sitemapped: true, | |
| zeroDataRetention: false, // not supported on v0 | |
| apiKeyId: chunk?.api_key_id ?? null, | |
| }, | |
| priority: jobPriority, | |
| }; | |
| }); | |
| await lockURLs( | |
| id, | |
| sc, | |
| jobs.map(x => x.data.url), | |
| logger, | |
| ); | |
| await addCrawlJobs( | |
| id, | |
| jobs.map(x => x.jobId), | |
| logger, | |
| ); | |
| for (const job of jobs) { | |
| // add with sentry instrumentation | |
| await addScrapeJob(job.data, job.jobId, job.priority); | |
| } | |
| }); | |
| if (sitemap === 0) { | |
| await lockURL(id, sc, url); | |
| // Not needed, first one should be 15. | |
| // const jobPriority = await getJobPriority({team_id, basePriority: 10}) | |
| const jobId = uuidv7(); | |
| await addScrapeJob( | |
| { | |
| url, | |
| mode: "single_urls", | |
| crawlerOptions, | |
| scrapeOptions, | |
| internalOptions, | |
| team_id, | |
| origin: req.body.origin ?? defaultOrigin, | |
| integration: req.body.integration, | |
| billing: { endpoint: "crawl", jobId: id }, | |
| crawl_id: id, | |
| zeroDataRetention: false, // not supported on v0 | |
| apiKeyId: chunk?.api_key_id ?? null, | |
| }, | |
| jobId, | |
| await getJobPriority({ team_id, basePriority: 15 }), | |
| ); | |
| await addCrawlJob(id, jobId, logger); | |
| } | |
| res.json({ jobId: id }); | |
| } catch (error) { | |
| Sentry.captureException(error); | |
| logger.error(error); | |
| return res.status(500).json({ | |
| error: error instanceof ZodError ? "Invalid URL" : error.message, | |
| }); | |
| } | |
| } | |