Spaces:
Sleeping
Sleeping
Download apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts from s-eto-hub/firecrawl-api: direct link, hf CLI and curl.
- Browser
- Download file 560 Bytes
-
https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
- Command line
-
hf download hf://spaces/s-eto-hub/firecrawl-api/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
-
curl -L -o shouldCheckRobots.ts https://huggingface.co/spaces/s-eto-hub/firecrawl-api/resolve/main/apps/api/src/scraper/scrapeURL/shouldCheckRobots.ts
560 Bytes
| import type { TeamFlags } from "../../controllers/v1/types"; | |
| // The invariant "lockdown never fetches robots.txt" is load-bearing for the | |
| // lockdown guarantee (robots.txt is a request to the target domain). Keep this | |
| // in its own file so it can be unit-tested without dragging in scrapeURL's | |
| // ESM-heavy module graph. | |
| export function shouldCheckRobots( | |
| options: { lockdown?: boolean }, | |
| internalOptions: { teamFlags?: TeamFlags }, | |
| ): boolean { | |
| if (options.lockdown) { | |
| return false; | |
| } | |
| return !!internalOptions.teamFlags?.checkRobotsOnScrape; | |
| } | |