File size: 21,028 Bytes
8de1fe2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f4dc314
 
 
 
 
 
 
 
 
 
 
 
8de1fe2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
// ============================================
// FETCH FUNCTIONALITY - ENHANCED WITH 403 HANDLING
// ============================================

import CONFIG from './config.js';

// fetch.js - Add this export at the end of the file
export async function fetchImage(url, options = {}) {
    try {
        options = options || {};

        // Input validation
        if (typeof url !== 'string' || url.trim() === '') {
            throw new Error('URL must be a non-empty string');
        }

        // Handle YouTube thumbnails specifically
        if (url && (url.includes('youtube.com') || url.includes('youtu.be'))) {
            try {
                // Extract video ID from various YouTube URL formats
                let videoId = '';
                const ytMatch = url.match(/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/\s]{11})/i);
                if (ytMatch && ytMatch[1]) {
                    videoId = ytMatch[1];
                    // Use the more reliable i.ytimg.com domain
                    url = `https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`;
                    console.log(`🔄 Converted YouTube URL to: ${url}`);
                }
            } catch (e) {
                console.warn('Error processing YouTube URL:', e);
            }
        }

        // Extract origin and domain for referer header and check if domain is problematic
        let origin = '';
        let domain = '';
        try {
            const urlObj = new URL(url);
            origin = urlObj.origin;
            domain = urlObj.hostname;

            // Check if domain is problematic before attempting to fetch
            if (CONFIG.isProblematicDomain(domain)) {
                console.log(`ℹ️ Using proxy for known problematic domain: ${domain}`);
                if (options.skipProxy) {
                    throw new Error('Skipping proxy for problematic domain as requested');
                }
                // Force proxy fallback by throwing an error that will trigger the proxy fallback logic
                throw new Error('Domain is marked as problematic, using proxy');
            }
        } catch (e) {
            // If URL parsing fails, continue with normal flow
            if (e.message === 'Domain is marked as problematic, using proxy') {
                throw e; // Re-throw to trigger proxy fallback
            }
            console.warn(`Warning parsing URL ${url}:`, e.message);
            origin = '';
            domain = '';
        }

        // Extract original request headers if provided
        const originalUserAgent = options.originalHeaders?.['user-agent'] || options.originalHeaders?.['User-Agent'];
        const originalReferer = options.originalHeaders?.['referer'] || options.originalHeaders?.['Referer'];
        const originalAccept = options.originalHeaders?.['accept'] || options.originalHeaders?.['Accept'];
        const originalAcceptLanguage = options.originalHeaders?.['accept-language'] || options.originalHeaders?.['Accept-Language'];

        // Check if we have any original headers to use
        const hasOriginalHeaders = originalUserAgent || originalReferer || originalAccept || originalAcceptLanguage;

        // Try multiple header configurations for better compatibility
        const headerSets = [
            // 0. ORIGINAL REQUEST HEADERS - Use exact headers from the original request (highest priority)
            ...(hasOriginalHeaders ? [{
                'User-Agent': originalUserAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': originalAccept || 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
                'Accept-Language': originalAcceptLanguage || 'en-US,en;q=0.9',
                'Referer': originalReferer || origin || url,
                'Origin': origin,
                'Connection': 'keep-alive',
                ...options.headers
            }] : []),

            // 1. EXACT DOMAIN MATCH - Referer matches origin (most likely to work)
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin || url, // Use exact origin
                'Origin': origin,
                ...options.headers
            },

            // 2. FIREFOX - Match the user's browser (from logs)
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0',
                'Accept': 'image/avif,image/webp,*/*',
                'Accept-Language': 'en-US,en;q=0.5',
                'Accept-Encoding': 'gzip, deflate, br',
                'Referer': origin || url,
                'Connection': 'keep-alive',
                'DNT': '1',
                ...options.headers
            },

            // 5. NO ACCEPT-ENCODING - Bypass compression-blocking
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': '*/*',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin,
                'Origin': origin,
                'Connection': 'keep-alive',
                ...options.headers
            },

            // 6. MOBILE CHROME - Different user agent class
            {
                'User-Agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
                'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin,
                'Connection': 'keep-alive',
                ...options.headers
            },

            // 7. ULTRA MINIMAL - Absolute bare minimum
            {
                'User-Agent': CONFIG.USER_AGENT,
                'Referer': origin,
                ...options.headers
            },

            // 8. SAME SITE - Pretend we're from the same domain
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin || url,
                'Origin': origin,
                'Sec-Fetch-Site': 'same-origin',
                'Sec-Fetch-Mode': 'no-cors',
                'Sec-Fetch-Dest': 'image',
                ...options.headers
            },

            // 9. MOBILE SAFARI (iOS)
            {
                'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1',
                'Accept': 'image/webp,image/*,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin,
                'Connection': 'keep-alive',
                ...options.headers
            },

            // 10. OLD CHROME - Sometimes older UA bypasses new protections
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
                'Accept': 'image/webp,image/apng,image/*,*/*',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin,
                ...options.headers
            },

            // 11. EDGE - Microsoft Edge user agent
            {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
                'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
                'Accept-Language': 'en-US,en;q=0.9',
                'Referer': origin,
                'Origin': origin,
                ...options.headers
            },

            // 12. FIREFOX LINUX - Different platform
            {
                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0',
                'Accept': 'image/avif,image/webp,*/*',
                'Accept-Language': 'en-US,en;q=0.5',
                'Referer': origin,
                'Connection': 'keep-alive',
                ...options.headers
            },

        ];

        let lastError;

        // Try each header set until one works
        for (let i = 0; i < headerSets.length; i++) {
            const controller = new AbortController();
            let timeoutId;

            try {
                let fetchSignal;
                if (options.signal) {
                    if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.any === 'function') {
                        fetchSignal = AbortSignal.any([options.signal, controller.signal]);
                    } else {
                        options.signal.addEventListener('abort', () => controller.abort(), { once: true });
                        if (options.signal.aborted) controller.abort();
                        fetchSignal = controller.signal;
                    }
                } else {
                    fetchSignal = controller.signal;
                }
                timeoutId = setTimeout(() => controller.abort(), CONFIG.FETCH_TIMEOUT);

                console.log(`🔄 Attempt ${i + 1}/${headerSets.length} with headers...`);
                const response = await fetch(url, {
                    method: 'GET',
                    headers: headerSets[i],
                    signal: fetchSignal
                });

                clearTimeout(timeoutId);

                if (response.ok) {
                    let buffer;
                    // Optimize: Use bytes() if available (Bun), otherwise fallback
                    if (typeof response.bytes === 'function') {
                        buffer = await response.bytes();
                    } else {
                        buffer = Buffer.from(await response.arrayBuffer());
                    }
                    console.log(`✅ Fetch successful with header set ${i + 1} for: ${url.substring(0, 60)}`);
                    return {
                        ok: true,
                        buffer,
                        contentType: response.headers.get('content-type') || 'image/jpeg'
                    };
                }

                // Log detailed error information
                console.error(`❌ Header set ${i + 1} failed: HTTP ${response.status} ${response.statusText}`);
                lastError = new Error(`HTTP error! status: ${response.status}`);

                // If it's a 403/401, mark domain as problematic
                if ((response.status === 403 || response.status === 401) && domain) {
                    console.log(`⚠️ Adding ${domain} to problematic domains due to HTTP ${response.status}`);
                    await CONFIG.addProblematicDomain(domain).catch(console.error);
                }

                // If it's not a 403/401, break (other errors are less likely to be fixed by headers)
                if (response.status !== 403 && response.status !== 401) {
                    break;
                }

                // Delay only after actual failure, not before next attempt
                if (i < headerSets.length - 1) {
                    const jitter = Math.floor(Math.random() * 500);
                    await new Promise(r => setTimeout(r, 500 + jitter));
                }

            } catch (error) {
                if (error.name === 'AbortError') {
                    console.warn(`⚠️ Request timeout (attempt ${i + 1}/${headerSets.length})`);
                    lastError = new Error('Request timeout');
                } else {
                    // If we have a domain and it's a network error, mark as problematic
                    if (domain && (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT')) {
                        console.log(`⚠️ Adding ${domain} to problematic domains due to ${error.code}`);
                        await CONFIG.addProblematicDomain(domain).catch(console.error);
                    }
                    console.warn(`⚠️ Request failed: ${error.message}`);
                    lastError = error;
                }

                // Add exponential backoff for retries
                if (i < headerSets.length - 1) {
                    const backoff = Math.min(1000 * Math.pow(2, i), 10000);
                    const jitter = Math.floor(Math.random() * 500);
                    await new Promise(r => setTimeout(r, backoff + jitter));
                }
            } finally {
                if (timeoutId) clearTimeout(timeoutId);
                if (!controller.signal.aborted) {
                    controller.abort();
                }
            }
        }

        // Proxy Fallback Logic (only if all header sets failed)
        const fallbackKey = domain && Object.keys(CONFIG.PROXY_FALLBACKS).find(key => domain.includes(key));
        if (fallbackKey && !options.fromProxy && lastError) {
            const fallbackProxy = CONFIG.PROXY_FALLBACKS[fallbackKey];

            // Skip if the fallback proxy is the same as the current host
            const currentHost = process.env.HOSTNAME ||
                process.env.SPACE_ID?.split('-')[0] ||
                'localhost';

            if (fallbackProxy.includes(currentHost)) {
                console.log(`⚠️ Skipping self-referencing proxy: ${fallbackProxy} (current host: ${currentHost})`);
                console.log(`ℹ️ Domain ${domain} will be served directly without proxy fallback`);
                return {
                    ok: false,
                    error: lastError?.message || 'All fetch attempts failed',
                    status: lastError?.status
                };
            }

            console.log(`🔄 Trying proxy fallback: ${fallbackProxy} for domain: ${domain}`);

            // Only try proxy if the original error was network-related, not HTTP errors
            if (!lastError.message.includes('HTTP error! status:')) {
                const proxyEndpoints = [
                    `${fallbackProxy}/?url=${encodeURIComponent(url)}`,
                    `${fallbackProxy}/proxy?url=${encodeURIComponent(url)}`,
                    `${fallbackProxy}/api/proxy?url=${encodeURIComponent(url)}`,
                    `${fallbackProxy}/?image=${encodeURIComponent(url)}`
                ];

                let lastProxyError = null;

                for (const proxyUrl of proxyEndpoints) {
                    const proxyController = new AbortController();
                    let proxyTimeoutId;

                    try {
                        const endpoint = proxyUrl.split('?')[0];
                        console.log(`🔄 Trying proxy endpoint: ${endpoint}`);

                        const proxyTimeout = Math.min(CONFIG.FETCH_TIMEOUT, 8000);
                        proxyTimeoutId = setTimeout(() => proxyController.abort(), proxyTimeout);

                        const response = await fetch(proxyUrl, {
                            method: 'GET',
                            signal: proxyController.signal,
                            headers: {
                                'User-Agent': CONFIG.USER_AGENT,
                                'Referer': new URL(proxyUrl).origin,
                                'X-Proxy-Attempt': '1/3'
                            }
                        });

                        clearTimeout(proxyTimeoutId);

                        if (response.ok) {
                            let buffer;
                            // Optimize: Use bytes() if available (Bun), otherwise fallback
                            if (typeof response.bytes === 'function') {
                                buffer = await response.bytes();
                            } else {
                                buffer = Buffer.from(await response.arrayBuffer());
                            }
                            console.log(`✅ Proxy fallback successful for: ${url.substring(0, 60)}`);
                            return {
                                ok: true,
                                buffer,
                                contentType: response.headers.get('content-type') || 'image/jpeg',
                                fromProxy: true
                            };
                        } else {
                            lastProxyError = `HTTP ${response.status} ${response.statusText}`;
                            console.warn(`⚠️ Proxy endpoint failed with status: ${lastProxyError}`);
                        }
                    } catch (error) {
                        if (error.name === 'AbortError') {
                            lastProxyError = 'Proxy request timeout';
                            console.warn(`⚠️ Proxy endpoint timeout: ${proxyUrl.split('?')[0]}`);
                        } else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
                            lastProxyError = `Proxy DNS/connection error: ${error.code}`;
                            console.warn(`⚠️ Proxy DNS/connection error: ${error.code}`);
                        } else {
                            lastProxyError = error.message;
                            console.warn(`⚠️ Proxy endpoint error: ${error.message}`);
                        }
                    } finally {
                        if (proxyTimeoutId) clearTimeout(proxyTimeoutId);
                        if (!proxyController.signal.aborted) {
                            proxyController.abort();
                        }
                    }
                }

                // If we get here, all proxy endpoints failed
                console.error(`❌ All proxy endpoints failed for ${url}. Last error: ${lastProxyError}`);
            }
        }

        // If we get here, all attempts including proxy fallback have failed
        return {
            ok: false,
            error: lastError?.message || 'All fetch attempts failed',
            status: lastError?.status || lastError?.response?.status || 500
        };
    } catch (error) {
        console.error('❌ Fetch error:', error.message);
        return {
            ok: false,
            error: error.message,
            status: error.status || error.response?.status || 500
        };
    }
}

/**

 * Validate image URL with enhanced security

 * @param {string} url - URL to validate

 * @returns {boolean} - Whether the URL is valid

 */
function isValidImageUrl(url) {
    if (typeof url !== 'string') return false;

    try {
        const { protocol, hostname, pathname, port, search } = new URL(url);

        // Only allow HTTP/HTTPS
        if (!['http:', 'https:'].includes(protocol)) {
            return false;
        }

        // Basic path validation
        if (!pathname || pathname === '/') {
            return false;
        }

        // Block private IP ranges and localhost
        const isLocalhost = (hostname === 'localhost' ||
            hostname === '[::1]' ||
            hostname.startsWith('127.') ||
            hostname.startsWith('192.168.') ||
            hostname.startsWith('10.') ||
            hostname.startsWith('172.') ||
            hostname.endsWith('.local'));
        if (isLocalhost) {
            return false;
        }

        // Block non-standard ports
        if (port && !['80', '443', '8080', '8443'].includes(port)) {
            return false;
        }

        // Check for common image file extensions
        const imageExtensions = /\.(jpg|jpeg|png|gif|webp|avif|svg|bmp|ico)(\?.*)?$/i;
        if (!imageExtensions.test(pathname) && !search.includes('format=')) {
            return false;
        }

        return true;
    } catch (error) {
        return false;
    }
}

/**

 * Safe URL decode

 * @param {string} str - String to decode

 * @returns {string} - Decoded string

 */
function safeDecodeURI(str) {
    try {
        return decodeURIComponent(str.replace(/%(?![0-9a-fA-F]{2})/g, '%25'));
    } catch (error) {
        return str;
    }
}

// Export all functions
export {
    isValidImageUrl,
    safeDecodeURI
};