Spaces:

Echo-AI-official
/

Fire-crawl

Paused

Fire-crawl / src /scraper /WebScraper /utils /maxDepthUtils.ts

Upload 280 files

0e759d2 verified over 1 year ago

430 Bytes

export function getAdjustedMaxDepth(
  url: string,
  maxCrawlDepth: number,
): number {
  const baseURLDepth = getURLDepth(url);
  const adjustedMaxDepth = maxCrawlDepth + baseURLDepth;
  return adjustedMaxDepth;
}

export function getURLDepth(url: string): number {
  const pathSplits = new URL(url).pathname
    .split("/")
    .filter((x) => x !== "" && x !== "index.php" && x !== "index.html");
  return pathSplits.length;
}

	export function getAdjustedMaxDepth(
	url: string,
	maxCrawlDepth: number,
	): number {
	const baseURLDepth = getURLDepth(url);
	const adjustedMaxDepth = maxCrawlDepth + baseURLDepth;
	return adjustedMaxDepth;
	}

	export function getURLDepth(url: string): number {
	const pathSplits = new URL(url).pathname
	.split("/")
	.filter((x) => x !== "" && x !== "index.php" && x !== "index.html");
	return pathSplits.length;
	}