// ============================================ // FETCH FUNCTIONALITY - ENHANCED WITH 403 HANDLING // ============================================ import CONFIG from './config.js'; // fetch.js - Add this export at the end of the file export async function fetchImage(url, options = {}) { try { options = options || {}; // Input validation if (typeof url !== 'string' || url.trim() === '') { throw new Error('URL must be a non-empty string'); } // Handle YouTube thumbnails specifically if (url && (url.includes('youtube.com') || url.includes('youtu.be'))) { try { // Extract video ID from various YouTube URL formats let videoId = ''; const ytMatch = url.match(/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/\s]{11})/i); if (ytMatch && ytMatch[1]) { videoId = ytMatch[1]; // Use the more reliable i.ytimg.com domain url = `https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`; console.log(`🔄 Converted YouTube URL to: ${url}`); } } catch (e) { console.warn('Error processing YouTube URL:', e); } } // Extract origin and domain for referer header and check if domain is problematic let origin = ''; let domain = ''; try { const urlObj = new URL(url); origin = urlObj.origin; domain = urlObj.hostname; // Check if domain is problematic before attempting to fetch if (CONFIG.isProblematicDomain(domain)) { console.log(`â„šī¸ Using proxy for known problematic domain: ${domain}`); if (options.skipProxy) { throw new Error('Skipping proxy for problematic domain as requested'); } // Force proxy fallback by throwing an error that will trigger the proxy fallback logic throw new Error('Domain is marked as problematic, using proxy'); } } catch (e) { // If URL parsing fails, continue with normal flow if (e.message === 'Domain is marked as problematic, using proxy') { throw e; // Re-throw to trigger proxy fallback } console.warn(`Warning parsing URL ${url}:`, e.message); origin = ''; domain = ''; } // Extract original request headers if provided const originalUserAgent = options.originalHeaders?.['user-agent'] || options.originalHeaders?.['User-Agent']; const originalReferer = options.originalHeaders?.['referer'] || options.originalHeaders?.['Referer']; const originalAccept = options.originalHeaders?.['accept'] || options.originalHeaders?.['Accept']; const originalAcceptLanguage = options.originalHeaders?.['accept-language'] || options.originalHeaders?.['Accept-Language']; // Check if we have any original headers to use const hasOriginalHeaders = originalUserAgent || originalReferer || originalAccept || originalAcceptLanguage; // Try multiple header configurations for better compatibility const headerSets = [ // 0. ORIGINAL REQUEST HEADERS - Use exact headers from the original request (highest priority) ...(hasOriginalHeaders ? [{ 'User-Agent': originalUserAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': originalAccept || 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Language': originalAcceptLanguage || 'en-US,en;q=0.9', 'Referer': originalReferer || origin || url, 'Origin': origin, 'Connection': 'keep-alive', ...options.headers }] : []), // 1. EXACT DOMAIN MATCH - Referer matches origin (most likely to work) { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin || url, // Use exact origin 'Origin': origin, ...options.headers }, // 2. FIREFOX - Match the user's browser (from logs) { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0', 'Accept': 'image/avif,image/webp,*/*', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'Referer': origin || url, 'Connection': 'keep-alive', 'DNT': '1', ...options.headers }, // 5. NO ACCEPT-ENCODING - Bypass compression-blocking { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin, 'Origin': origin, 'Connection': 'keep-alive', ...options.headers }, // 6. MOBILE CHROME - Different user agent class { 'User-Agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36', 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin, 'Connection': 'keep-alive', ...options.headers }, // 7. ULTRA MINIMAL - Absolute bare minimum { 'User-Agent': CONFIG.USER_AGENT, 'Referer': origin, ...options.headers }, // 8. SAME SITE - Pretend we're from the same domain { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin || url, 'Origin': origin, 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-Mode': 'no-cors', 'Sec-Fetch-Dest': 'image', ...options.headers }, // 9. MOBILE SAFARI (iOS) { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1', 'Accept': 'image/webp,image/*,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin, 'Connection': 'keep-alive', ...options.headers }, // 10. OLD CHROME - Sometimes older UA bypasses new protections { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36', 'Accept': 'image/webp,image/apng,image/*,*/*', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin, ...options.headers }, // 11. EDGE - Microsoft Edge user agent { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0', 'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': origin, 'Origin': origin, ...options.headers }, // 12. FIREFOX LINUX - Different platform { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0', 'Accept': 'image/avif,image/webp,*/*', 'Accept-Language': 'en-US,en;q=0.5', 'Referer': origin, 'Connection': 'keep-alive', ...options.headers }, ]; let lastError; // Try each header set until one works for (let i = 0; i < headerSets.length; i++) { const controller = new AbortController(); let timeoutId; try { let fetchSignal; if (options.signal) { if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.any === 'function') { fetchSignal = AbortSignal.any([options.signal, controller.signal]); } else { options.signal.addEventListener('abort', () => controller.abort(), { once: true }); if (options.signal.aborted) controller.abort(); fetchSignal = controller.signal; } } else { fetchSignal = controller.signal; } timeoutId = setTimeout(() => controller.abort(), CONFIG.FETCH_TIMEOUT); console.log(`🔄 Attempt ${i + 1}/${headerSets.length} with headers...`); const response = await fetch(url, { method: 'GET', headers: headerSets[i], signal: fetchSignal }); clearTimeout(timeoutId); if (response.ok) { let buffer; // Optimize: Use bytes() if available (Bun), otherwise fallback if (typeof response.bytes === 'function') { buffer = await response.bytes(); } else { buffer = Buffer.from(await response.arrayBuffer()); } console.log(`✅ Fetch successful with header set ${i + 1} for: ${url.substring(0, 60)}`); return { ok: true, buffer, contentType: response.headers.get('content-type') || 'image/jpeg' }; } // Log detailed error information console.error(`❌ Header set ${i + 1} failed: HTTP ${response.status} ${response.statusText}`); lastError = new Error(`HTTP error! status: ${response.status}`); // If it's a 403/401, mark domain as problematic if ((response.status === 403 || response.status === 401) && domain) { console.log(`âš ī¸ Adding ${domain} to problematic domains due to HTTP ${response.status}`); await CONFIG.addProblematicDomain(domain).catch(console.error); } // If it's not a 403/401, break (other errors are less likely to be fixed by headers) if (response.status !== 403 && response.status !== 401) { break; } // Delay only after actual failure, not before next attempt if (i < headerSets.length - 1) { const jitter = Math.floor(Math.random() * 500); await new Promise(r => setTimeout(r, 500 + jitter)); } } catch (error) { if (error.name === 'AbortError') { console.warn(`âš ī¸ Request timeout (attempt ${i + 1}/${headerSets.length})`); lastError = new Error('Request timeout'); } else { // If we have a domain and it's a network error, mark as problematic if (domain && (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT')) { console.log(`âš ī¸ Adding ${domain} to problematic domains due to ${error.code}`); await CONFIG.addProblematicDomain(domain).catch(console.error); } console.warn(`âš ī¸ Request failed: ${error.message}`); lastError = error; } // Add exponential backoff for retries if (i < headerSets.length - 1) { const backoff = Math.min(1000 * Math.pow(2, i), 10000); const jitter = Math.floor(Math.random() * 500); await new Promise(r => setTimeout(r, backoff + jitter)); } } finally { if (timeoutId) clearTimeout(timeoutId); if (!controller.signal.aborted) { controller.abort(); } } } // Proxy Fallback Logic (only if all header sets failed) const fallbackKey = domain && Object.keys(CONFIG.PROXY_FALLBACKS).find(key => domain.includes(key)); if (fallbackKey && !options.fromProxy && lastError) { const fallbackProxy = CONFIG.PROXY_FALLBACKS[fallbackKey]; // Skip if the fallback proxy is the same as the current host const currentHost = process.env.HOSTNAME || process.env.SPACE_ID?.split('-')[0] || 'localhost'; if (fallbackProxy.includes(currentHost)) { console.log(`âš ī¸ Skipping self-referencing proxy: ${fallbackProxy} (current host: ${currentHost})`); console.log(`â„šī¸ Domain ${domain} will be served directly without proxy fallback`); return { ok: false, error: lastError?.message || 'All fetch attempts failed', status: lastError?.status }; } console.log(`🔄 Trying proxy fallback: ${fallbackProxy} for domain: ${domain}`); // Only try proxy if the original error was network-related, not HTTP errors if (!lastError.message.includes('HTTP error! status:')) { const proxyEndpoints = [ `${fallbackProxy}/?url=${encodeURIComponent(url)}`, `${fallbackProxy}/proxy?url=${encodeURIComponent(url)}`, `${fallbackProxy}/api/proxy?url=${encodeURIComponent(url)}`, `${fallbackProxy}/?image=${encodeURIComponent(url)}` ]; let lastProxyError = null; for (const proxyUrl of proxyEndpoints) { const proxyController = new AbortController(); let proxyTimeoutId; try { const endpoint = proxyUrl.split('?')[0]; console.log(`🔄 Trying proxy endpoint: ${endpoint}`); const proxyTimeout = Math.min(CONFIG.FETCH_TIMEOUT, 8000); proxyTimeoutId = setTimeout(() => proxyController.abort(), proxyTimeout); const response = await fetch(proxyUrl, { method: 'GET', signal: proxyController.signal, headers: { 'User-Agent': CONFIG.USER_AGENT, 'Referer': new URL(proxyUrl).origin, 'X-Proxy-Attempt': '1/3' } }); clearTimeout(proxyTimeoutId); if (response.ok) { let buffer; // Optimize: Use bytes() if available (Bun), otherwise fallback if (typeof response.bytes === 'function') { buffer = await response.bytes(); } else { buffer = Buffer.from(await response.arrayBuffer()); } console.log(`✅ Proxy fallback successful for: ${url.substring(0, 60)}`); return { ok: true, buffer, contentType: response.headers.get('content-type') || 'image/jpeg', fromProxy: true }; } else { lastProxyError = `HTTP ${response.status} ${response.statusText}`; console.warn(`âš ī¸ Proxy endpoint failed with status: ${lastProxyError}`); } } catch (error) { if (error.name === 'AbortError') { lastProxyError = 'Proxy request timeout'; console.warn(`âš ī¸ Proxy endpoint timeout: ${proxyUrl.split('?')[0]}`); } else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { lastProxyError = `Proxy DNS/connection error: ${error.code}`; console.warn(`âš ī¸ Proxy DNS/connection error: ${error.code}`); } else { lastProxyError = error.message; console.warn(`âš ī¸ Proxy endpoint error: ${error.message}`); } } finally { if (proxyTimeoutId) clearTimeout(proxyTimeoutId); if (!proxyController.signal.aborted) { proxyController.abort(); } } } // If we get here, all proxy endpoints failed console.error(`❌ All proxy endpoints failed for ${url}. Last error: ${lastProxyError}`); } } // If we get here, all attempts including proxy fallback have failed return { ok: false, error: lastError?.message || 'All fetch attempts failed', status: lastError?.status || lastError?.response?.status || 500 }; } catch (error) { console.error('❌ Fetch error:', error.message); return { ok: false, error: error.message, status: error.status || error.response?.status || 500 }; } } /** * Validate image URL with enhanced security * @param {string} url - URL to validate * @returns {boolean} - Whether the URL is valid */ function isValidImageUrl(url) { if (typeof url !== 'string') return false; try { const { protocol, hostname, pathname, port, search } = new URL(url); // Only allow HTTP/HTTPS if (!['http:', 'https:'].includes(protocol)) { return false; } // Basic path validation if (!pathname || pathname === '/') { return false; } // Block private IP ranges and localhost const isLocalhost = (hostname === 'localhost' || hostname === '[::1]' || hostname.startsWith('127.') || hostname.startsWith('192.168.') || hostname.startsWith('10.') || hostname.startsWith('172.') || hostname.endsWith('.local')); if (isLocalhost) { return false; } // Block non-standard ports if (port && !['80', '443', '8080', '8443'].includes(port)) { return false; } // Check for common image file extensions const imageExtensions = /\.(jpg|jpeg|png|gif|webp|avif|svg|bmp|ico)(\?.*)?$/i; if (!imageExtensions.test(pathname) && !search.includes('format=')) { return false; } return true; } catch (error) { return false; } } /** * Safe URL decode * @param {string} str - String to decode * @returns {string} - Decoded string */ function safeDecodeURI(str) { try { return decodeURIComponent(str.replace(/%(?![0-9a-fA-F]{2})/g, '%25')); } catch (error) { return str; } } // Export all functions export { isValidImageUrl, safeDecodeURI };