pixelpinch-proxy / src /fetch.js
Pixel Pinch Extensions
feat: implement a Bun-based image proxy server with robust fetching, compression, and format conversion capabilities.
f4dc314
Raw History Blame Contribute Delete
21 kB
// ============================================
// FETCH FUNCTIONALITY - ENHANCED WITH 403 HANDLING
// ============================================
import CONFIG from './config.js';
// fetch.js - Add this export at the end of the file
export async function fetchImage(url, options = {}) {
try {
options = options || {};
// Input validation
if (typeof url !== 'string' || url.trim() === '') {
throw new Error('URL must be a non-empty string');
}
// Handle YouTube thumbnails specifically
if (url && (url.includes('youtube.com') || url.includes('youtu.be'))) {
try {
// Extract video ID from various YouTube URL formats
let videoId = '';
const ytMatch = url.match(/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/\s]{11})/i);
if (ytMatch && ytMatch[1]) {
videoId = ytMatch[1];
// Use the more reliable i.ytimg.com domain
url = `https://i.ytimg.com/vi/${videoId}/maxresdefault.jpg`;
console.log(`🔄 Converted YouTube URL to: ${url}`);
}
} catch (e) {
console.warn('Error processing YouTube URL:', e);
}
}
// Extract origin and domain for referer header and check if domain is problematic
let origin = '';
let domain = '';
try {
const urlObj = new URL(url);
origin = urlObj.origin;
domain = urlObj.hostname;
// Check if domain is problematic before attempting to fetch
if (CONFIG.isProblematicDomain(domain)) {
console.log(`ℹ️ Using proxy for known problematic domain: ${domain}`);
if (options.skipProxy) {
throw new Error('Skipping proxy for problematic domain as requested');
}
// Force proxy fallback by throwing an error that will trigger the proxy fallback logic
throw new Error('Domain is marked as problematic, using proxy');
}
} catch (e) {
// If URL parsing fails, continue with normal flow
if (e.message === 'Domain is marked as problematic, using proxy') {
throw e; // Re-throw to trigger proxy fallback
}
console.warn(`Warning parsing URL ${url}:`, e.message);
origin = '';
domain = '';
}
// Extract original request headers if provided
const originalUserAgent = options.originalHeaders?.['user-agent'] || options.originalHeaders?.['User-Agent'];
const originalReferer = options.originalHeaders?.['referer'] || options.originalHeaders?.['Referer'];
const originalAccept = options.originalHeaders?.['accept'] || options.originalHeaders?.['Accept'];
const originalAcceptLanguage = options.originalHeaders?.['accept-language'] || options.originalHeaders?.['Accept-Language'];
// Check if we have any original headers to use
const hasOriginalHeaders = originalUserAgent || originalReferer || originalAccept || originalAcceptLanguage;
// Try multiple header configurations for better compatibility
const headerSets = [
// 0. ORIGINAL REQUEST HEADERS - Use exact headers from the original request (highest priority)
...(hasOriginalHeaders ? [{
'User-Agent': originalUserAgent || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': originalAccept || 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': originalAcceptLanguage || 'en-US,en;q=0.9',
'Referer': originalReferer || origin || url,
'Origin': origin,
'Connection': 'keep-alive',
...options.headers
}] : []),
// 1. EXACT DOMAIN MATCH - Referer matches origin (most likely to work)
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin || url, // Use exact origin
'Origin': origin,
...options.headers
},
// 2. FIREFOX - Match the user's browser (from logs)
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0',
'Accept': 'image/avif,image/webp,*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': origin || url,
'Connection': 'keep-alive',
'DNT': '1',
...options.headers
},
// 5. NO ACCEPT-ENCODING - Bypass compression-blocking
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin,
'Origin': origin,
'Connection': 'keep-alive',
...options.headers
},
// 6. MOBILE CHROME - Different user agent class
{
'User-Agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin,
'Connection': 'keep-alive',
...options.headers
},
// 7. ULTRA MINIMAL - Absolute bare minimum
{
'User-Agent': CONFIG.USER_AGENT,
'Referer': origin,
...options.headers
},
// 8. SAME SITE - Pretend we're from the same domain
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin || url,
'Origin': origin,
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'no-cors',
'Sec-Fetch-Dest': 'image',
...options.headers
},
// 9. MOBILE SAFARI (iOS)
{
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1',
'Accept': 'image/webp,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin,
'Connection': 'keep-alive',
...options.headers
},
// 10. OLD CHROME - Sometimes older UA bypasses new protections
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
'Accept': 'image/webp,image/apng,image/*,*/*',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin,
...options.headers
},
// 11. EDGE - Microsoft Edge user agent
{
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Referer': origin,
'Origin': origin,
...options.headers
},
// 12. FIREFOX LINUX - Different platform
{
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:145.0) Gecko/20100101 Firefox/145.0',
'Accept': 'image/avif,image/webp,*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Referer': origin,
'Connection': 'keep-alive',
...options.headers
},
];
let lastError;
// Try each header set until one works
for (let i = 0; i < headerSets.length; i++) {
const controller = new AbortController();
let timeoutId;
try {
let fetchSignal;
if (options.signal) {
if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.any === 'function') {
fetchSignal = AbortSignal.any([options.signal, controller.signal]);
} else {
options.signal.addEventListener('abort', () => controller.abort(), { once: true });
if (options.signal.aborted) controller.abort();
fetchSignal = controller.signal;
}
} else {
fetchSignal = controller.signal;
}
timeoutId = setTimeout(() => controller.abort(), CONFIG.FETCH_TIMEOUT);
console.log(`🔄 Attempt ${i + 1}/${headerSets.length} with headers...`);
const response = await fetch(url, {
method: 'GET',
headers: headerSets[i],
signal: fetchSignal
});
clearTimeout(timeoutId);
if (response.ok) {
let buffer;
// Optimize: Use bytes() if available (Bun), otherwise fallback
if (typeof response.bytes === 'function') {
buffer = await response.bytes();
} else {
buffer = Buffer.from(await response.arrayBuffer());
}
console.log(`✅ Fetch successful with header set ${i + 1} for: ${url.substring(0, 60)}`);
return {
ok: true,
buffer,
contentType: response.headers.get('content-type') || 'image/jpeg'
};
}
// Log detailed error information
console.error(`❌ Header set ${i + 1} failed: HTTP ${response.status} ${response.statusText}`);
lastError = new Error(`HTTP error! status: ${response.status}`);
// If it's a 403/401, mark domain as problematic
if ((response.status === 403 || response.status === 401) && domain) {
console.log(`⚠️ Adding ${domain} to problematic domains due to HTTP ${response.status}`);
await CONFIG.addProblematicDomain(domain).catch(console.error);
}
// If it's not a 403/401, break (other errors are less likely to be fixed by headers)
if (response.status !== 403 && response.status !== 401) {
break;
}
// Delay only after actual failure, not before next attempt
if (i < headerSets.length - 1) {
const jitter = Math.floor(Math.random() * 500);
await new Promise(r => setTimeout(r, 500 + jitter));
}
} catch (error) {
if (error.name === 'AbortError') {
console.warn(`⚠️ Request timeout (attempt ${i + 1}/${headerSets.length})`);
lastError = new Error('Request timeout');
} else {
// If we have a domain and it's a network error, mark as problematic
if (domain && (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED' || error.code === 'ETIMEDOUT')) {
console.log(`⚠️ Adding ${domain} to problematic domains due to ${error.code}`);
await CONFIG.addProblematicDomain(domain).catch(console.error);
}
console.warn(`⚠️ Request failed: ${error.message}`);
lastError = error;
}
// Add exponential backoff for retries
if (i < headerSets.length - 1) {
const backoff = Math.min(1000 * Math.pow(2, i), 10000);
const jitter = Math.floor(Math.random() * 500);
await new Promise(r => setTimeout(r, backoff + jitter));
}
} finally {
if (timeoutId) clearTimeout(timeoutId);
if (!controller.signal.aborted) {
controller.abort();
}
}
}
// Proxy Fallback Logic (only if all header sets failed)
const fallbackKey = domain && Object.keys(CONFIG.PROXY_FALLBACKS).find(key => domain.includes(key));
if (fallbackKey && !options.fromProxy && lastError) {
const fallbackProxy = CONFIG.PROXY_FALLBACKS[fallbackKey];
// Skip if the fallback proxy is the same as the current host
const currentHost = process.env.HOSTNAME ||
process.env.SPACE_ID?.split('-')[0] ||
'localhost';
if (fallbackProxy.includes(currentHost)) {
console.log(`⚠️ Skipping self-referencing proxy: ${fallbackProxy} (current host: ${currentHost})`);
console.log(`ℹ️ Domain ${domain} will be served directly without proxy fallback`);
return {
ok: false,
error: lastError?.message || 'All fetch attempts failed',
status: lastError?.status
};
}
console.log(`🔄 Trying proxy fallback: ${fallbackProxy} for domain: ${domain}`);
// Only try proxy if the original error was network-related, not HTTP errors
if (!lastError.message.includes('HTTP error! status:')) {
const proxyEndpoints = [
`${fallbackProxy}/?url=${encodeURIComponent(url)}`,
`${fallbackProxy}/proxy?url=${encodeURIComponent(url)}`,
`${fallbackProxy}/api/proxy?url=${encodeURIComponent(url)}`,
`${fallbackProxy}/?image=${encodeURIComponent(url)}`
];
let lastProxyError = null;
for (const proxyUrl of proxyEndpoints) {
const proxyController = new AbortController();
let proxyTimeoutId;
try {
const endpoint = proxyUrl.split('?')[0];
console.log(`🔄 Trying proxy endpoint: ${endpoint}`);
const proxyTimeout = Math.min(CONFIG.FETCH_TIMEOUT, 8000);
proxyTimeoutId = setTimeout(() => proxyController.abort(), proxyTimeout);
const response = await fetch(proxyUrl, {
method: 'GET',
signal: proxyController.signal,
headers: {
'User-Agent': CONFIG.USER_AGENT,
'Referer': new URL(proxyUrl).origin,
'X-Proxy-Attempt': '1/3'
}
});
clearTimeout(proxyTimeoutId);
if (response.ok) {
let buffer;
// Optimize: Use bytes() if available (Bun), otherwise fallback
if (typeof response.bytes === 'function') {
buffer = await response.bytes();
} else {
buffer = Buffer.from(await response.arrayBuffer());
}
console.log(`✅ Proxy fallback successful for: ${url.substring(0, 60)}`);
return {
ok: true,
buffer,
contentType: response.headers.get('content-type') || 'image/jpeg',
fromProxy: true
};
} else {
lastProxyError = `HTTP ${response.status} ${response.statusText}`;
console.warn(`⚠️ Proxy endpoint failed with status: ${lastProxyError}`);
}
} catch (error) {
if (error.name === 'AbortError') {
lastProxyError = 'Proxy request timeout';
console.warn(`⚠️ Proxy endpoint timeout: ${proxyUrl.split('?')[0]}`);
} else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
lastProxyError = `Proxy DNS/connection error: ${error.code}`;
console.warn(`⚠️ Proxy DNS/connection error: ${error.code}`);
} else {
lastProxyError = error.message;
console.warn(`⚠️ Proxy endpoint error: ${error.message}`);
}
} finally {
if (proxyTimeoutId) clearTimeout(proxyTimeoutId);
if (!proxyController.signal.aborted) {
proxyController.abort();
}
}
}
// If we get here, all proxy endpoints failed
console.error(`❌ All proxy endpoints failed for ${url}. Last error: ${lastProxyError}`);
}
}
// If we get here, all attempts including proxy fallback have failed
return {
ok: false,
error: lastError?.message || 'All fetch attempts failed',
status: lastError?.status || lastError?.response?.status || 500
};
} catch (error) {
console.error('❌ Fetch error:', error.message);
return {
ok: false,
error: error.message,
status: error.status || error.response?.status || 500
};
}
}
/**
* Validate image URL with enhanced security
* @param {string} url - URL to validate
* @returns {boolean} - Whether the URL is valid
*/
function isValidImageUrl(url) {
if (typeof url !== 'string') return false;
try {
const { protocol, hostname, pathname, port, search } = new URL(url);
// Only allow HTTP/HTTPS
if (!['http:', 'https:'].includes(protocol)) {
return false;
}
// Basic path validation
if (!pathname || pathname === '/') {
return false;
}
// Block private IP ranges and localhost
const isLocalhost = (hostname === 'localhost' ||
hostname === '[::1]' ||
hostname.startsWith('127.') ||
hostname.startsWith('192.168.') ||
hostname.startsWith('10.') ||
hostname.startsWith('172.') ||
hostname.endsWith('.local'));
if (isLocalhost) {
return false;
}
// Block non-standard ports
if (port && !['80', '443', '8080', '8443'].includes(port)) {
return false;
}
// Check for common image file extensions
const imageExtensions = /\.(jpg|jpeg|png|gif|webp|avif|svg|bmp|ico)(\?.*)?$/i;
if (!imageExtensions.test(pathname) && !search.includes('format=')) {
return false;
}
return true;
} catch (error) {
return false;
}
}
/**
* Safe URL decode
* @param {string} str - String to decode
* @returns {string} - Decoded string
*/
function safeDecodeURI(str) {
try {
return decodeURIComponent(str.replace(/%(?![0-9a-fA-F]{2})/g, '%25'));
} catch (error) {
return str;
}
}
// Export all functions
export {
isValidImageUrl,
safeDecodeURI
};