Spaces:
Sleeping
Sleeping
File size: 2,522 Bytes
097eaf0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | import { configDotenv } from "dotenv";
import { parse } from "tldts";
import { Engine } from "../../scrapeURL/engines";
import { logger } from "../../../lib/logger";
import { config } from "../../../config";
configDotenv();
type EngineForcingMapping = {
[domain: string]: Engine | Engine[];
};
let engineMappings: EngineForcingMapping | null = null;
/**
* Initialize the engine forcing mappings from environment variable
* Expected format: JSON object with domain patterns as keys and engines as values
* Example: {"example.com": "playwright", "*.google.com": ["fire-engine;chrome-cdp", "playwright"]}
*/
export function initializeEngineForcing() {
const envVar = config.FORCED_ENGINE_DOMAINS;
if (!envVar || envVar.trim() === "") {
engineMappings = {};
return;
}
try {
engineMappings = JSON.parse(envVar);
} catch (error) {
logger.error("Error parsing FORCED_ENGINE_DOMAINS environment variable", {
error,
});
engineMappings = {};
}
}
/**
* Check if a domain matches a pattern (supports wildcards)
* @param domain The domain to check (e.g., "sub.example.com")
* @param pattern The pattern to match against (e.g., "example.com" or "*.example.com")
* @returns true if the domain matches the pattern
*/
function domainMatchesPattern(domain: string, pattern: string): boolean {
if (domain === pattern) {
return true;
}
if (pattern.startsWith("*.")) {
const basePattern = pattern.substring(2); // Remove "*."
return domain === basePattern || domain.endsWith(`.${basePattern}`);
}
return domain === pattern || domain.endsWith(`.${pattern}`);
}
/**
* Get the forced engine(s) for a given URL based on domain mappings
* @param url The URL to check
* @returns The forced engine(s) if a match is found, undefined otherwise
*/
export function getEngineForUrl(url: string): Engine | Engine[] | undefined {
if (engineMappings === null) {
return undefined;
}
if (Object.keys(engineMappings).length === 0) {
return undefined;
}
const lowerCaseUrl = url.trim().toLowerCase();
let parsedUrl: any;
try {
parsedUrl = parse(lowerCaseUrl);
} catch (error) {
logger.warn("Error parsing URL for engine forcing", { url, error });
return undefined;
}
const domain = parsedUrl.domain;
if (!domain) {
return undefined;
}
for (const [pattern, engine] of Object.entries(engineMappings)) {
if (domainMatchesPattern(domain, pattern.toLowerCase())) {
return engine;
}
}
return undefined;
}
|