Spaces:
Sleeping
Sleeping
File size: 5,378 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | // crawler.test.ts
import { WebCrawler } from "../crawler";
import axios from "axios";
import robotsParser from "robots-parser";
jest.mock("axios");
jest.mock("robots-parser");
describe("WebCrawler", () => {
let crawler: WebCrawler;
const mockAxios = axios as jest.Mocked<typeof axios>;
const mockRobotsParser = robotsParser as jest.MockedFunction<
typeof robotsParser
>;
let maxCrawledDepth: number;
beforeEach(() => {
// Setup default mocks
mockAxios.get.mockImplementation(url => {
if (url.includes("robots.txt")) {
return Promise.resolve({ data: "User-agent: *\nAllow: /" });
} else if (url.includes("sitemap.xml")) {
return Promise.resolve({ data: "sitemap content" }); // You would normally parse this to URLs
}
return Promise.resolve({ data: "<html></html>" });
});
mockRobotsParser.mockReturnValue({
isAllowed: jest.fn().mockReturnValue(true),
isDisallowed: jest.fn().mockReturnValue(false),
getMatchingLineNumber: jest.fn().mockReturnValue(0),
getCrawlDelay: jest.fn().mockReturnValue(0),
getSitemaps: jest.fn().mockReturnValue([]),
getPreferredHost: jest.fn().mockReturnValue("example.com"),
});
});
it("should respect the limit parameter by not returning more links than specified", async () => {
const initialUrl = "http://example.com";
const limit = 2; // Set a limit for the number of links
crawler = new WebCrawler({
jobId: "TEST",
initialUrl: initialUrl,
includes: [],
excludes: [],
limit: limit, // Apply the limit
maxCrawledDepth: 10,
});
// Mock sitemap fetching function to return more links than the limit
crawler["tryFetchSitemapLinks"] = jest
.fn()
.mockResolvedValue([
initialUrl,
initialUrl + "/page1",
initialUrl + "/page2",
initialUrl + "/page3",
]);
const filteredLinks = await crawler["filterLinks"](
[
initialUrl,
initialUrl + "/page1",
initialUrl + "/page2",
initialUrl + "/page3",
],
limit,
10,
);
expect(filteredLinks.links.length).toBe(limit); // Check if the number of results respects the limit
expect(filteredLinks.links).toEqual([initialUrl, initialUrl + "/page1"]);
});
it("should filter subdomain URLs with includePaths when allowSubdomains is true", async () => {
const initialUrl = "https://example.com";
crawler = new WebCrawler({
jobId: "TEST",
initialUrl: initialUrl,
includes: ["^/pricing$"], // Only allow /pricing path
excludes: [],
limit: 10,
maxCrawledDepth: 10,
allowSubdomains: true,
});
const linksToFilter = [
"https://example.com/pricing", // Should pass: base domain + /pricing
"https://example.com/blog", // Should fail: base domain but wrong path
"https://sub.example.com/pricing", // Should pass: subdomain + /pricing
"https://sub.example.com/about", // Should fail: subdomain but wrong path
"https://other.example.com/pricing", // Should pass: subdomain + /pricing
"https://other.example.com/contact", // Should fail: subdomain but wrong path
];
const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);
expect(filteredLinks.links.length).toBe(3);
expect(filteredLinks.links).toContain("https://example.com/pricing");
expect(filteredLinks.links).toContain("https://sub.example.com/pricing");
expect(filteredLinks.links).toContain("https://other.example.com/pricing");
// Verify denied links
expect(filteredLinks.denialReasons.has("https://example.com/blog")).toBe(
true,
);
expect(
filteredLinks.denialReasons.has("https://sub.example.com/about"),
).toBe(true);
expect(
filteredLinks.denialReasons.has("https://other.example.com/contact"),
).toBe(true);
});
it("should filter subdomain URLs with includePaths using regexOnFullURL", async () => {
const initialUrl = "https://example.com";
crawler = new WebCrawler({
jobId: "TEST",
initialUrl: initialUrl,
includes: ["^https://([a-z0-9-]+\\.)?example\\.com/pricing$"], // Full URL pattern
excludes: [],
limit: 10,
maxCrawledDepth: 10,
allowSubdomains: true,
regexOnFullURL: true,
});
const linksToFilter = [
"https://example.com/pricing", // Should pass: matches pattern
"https://example.com/pricing/details", // Should fail: doesn't match exact pattern
"https://sub.example.com/pricing", // Should pass: subdomain + /pricing
"https://api.example.com/pricing", // Should pass: subdomain + /pricing
"https://sub.example.com/blog", // Should fail: wrong path
];
const filteredLinks = await crawler["filterLinks"](linksToFilter, 10, 10);
expect(filteredLinks.links.length).toBe(3);
expect(filteredLinks.links).toContain("https://example.com/pricing");
expect(filteredLinks.links).toContain("https://sub.example.com/pricing");
expect(filteredLinks.links).toContain("https://api.example.com/pricing");
// Verify denied links
expect(
filteredLinks.denialReasons.has("https://example.com/pricing/details"),
).toBe(true);
expect(
filteredLinks.denialReasons.has("https://sub.example.com/blog"),
).toBe(true);
});
});
|