Spaces:
Sleeping
Sleeping
File size: 560 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | import type { TeamFlags } from "../../controllers/v1/types";
// The invariant "lockdown never fetches robots.txt" is load-bearing for the
// lockdown guarantee (robots.txt is a request to the target domain). Keep this
// in its own file so it can be unit-tested without dragging in scrapeURL's
// ESM-heavy module graph.
export function shouldCheckRobots(
options: { lockdown?: boolean },
internalOptions: { teamFlags?: TeamFlags },
): boolean {
if (options.lockdown) {
return false;
}
return !!internalOptions.teamFlags?.checkRobotsOnScrape;
}
|