firecrawl-api / apps /api /src /scraper /scrapeURL /shouldCheckRobots.ts
s-eto-hub's picture
Commit de nettoyage des binaires
097eaf0
Raw History Blame Contribute Delete
560 Bytes
import type { TeamFlags } from "../../controllers/v1/types";
// The invariant "lockdown never fetches robots.txt" is load-bearing for the
// lockdown guarantee (robots.txt is a request to the target domain). Keep this
// in its own file so it can be unit-tested without dragging in scrapeURL's
// ESM-heavy module graph.
export function shouldCheckRobots(
options: { lockdown?: boolean },
internalOptions: { teamFlags?: TeamFlags },
): boolean {
if (options.lockdown) {
return false;
}
return !!internalOptions.teamFlags?.checkRobotsOnScrape;
}