Update index.js

index.js

@@ -4,6 +4,7 @@ const cheerio = require('cheerio');
 const AdmZip  = require('adm-zip');
 const https   = require('https');
 const http    = require('http');
+const http2   = require('http2');
 const zlib    = require('zlib');
 const { URL } = require('url');
  
@@ -21,9 +22,8 @@ app.use((req, res, next) => {
 // ================================================================
 //  PROXY CONFIG  ← edit these three lines to swap proxies
 //  Each takes a URL-ENCODED target appended to it. Set to '' to disable a slot.
-//  Dropped corsproxy.io (now requires an API key for programmatic use) and
-//  thingproxy.freeboard.io (dead). These two are the ones already proven in
-//  your SubDL Workers addon.
+//  PROXY_3 is a good slot for your own Cloudflare Worker relay — Worker egress
+//  already reaches CF-fronted subtitle sites in your SubDL addon.
 // ================================================================
 const PROXY_1 = 'https://api.codetabs.com/v1/proxy?quest=';
 const PROXY_2 = 'https://api.allorigins.win/raw?url=';
@@ -33,9 +33,12 @@ const PROXY_3 = '';
 // ================================================================
 //  DOWNLOAD TUNING  ← edit these to change retry/pacing behaviour
 // ================================================================
-const DL_RETRY_DELAY = 3000;  // ms between download attempts (404/503 backoff)
+const DL_RETRY_DELAY = 2000;  // ms between download attempts (404/503 backoff)
 const DL_STEP_DELAY  = 600;   // ms between detail-page fetch and download fetch
-const DL_ATTEMPTS    = 3;     // direct attempts before falling back to proxies
+const DL_ATTEMPTS    = 2;     // attempts per transport before moving to the next
+const DL_HARD_TIMEOUT = 12000;// ms per download attempt — fail fast, don't hang
+const TRY_HTTP2      = true;  // attempt HTTP/2 (browsers use h2; node https is h1)
+const TRY_INSECURE   = true;  // retry ignoring TLS verification errors
 const DEBUG_ROUTE    = true;  // expose GET /debug/:enc — set false to disable
 // ================================================================
  
@@ -167,7 +170,7 @@ const diag = {
   extractFailed: 0,         // got a body back but couldn't pull a subtitle out of it
   downloadsOK: 0,
   lastFailStage: null,      // 'detail' | 'button' | 'fetch' | 'extract'
-  lastStatuses: [],         // real HTTP status codes from the last download attempt
+  lastStatuses: [],         // per-transport attempt records from the last download
   lastArchiveKind: null,    // 'zip' | 'rar' | '7z' | 'gzip' | 'html' | 'unknown'
   lastNote: null,           // human-readable reason the last attempt died
 };
@@ -206,9 +209,10 @@ function _cookiesFrom(res, prior) {
 // identical from the outside.
 function _request(urlStr, opts = {}, redirects = 0, jar = '') {
   return new Promise((resolve, reject) => {
-    if (!urlStr || redirects > MAX_REDIRECTS) return reject(new Error('Invalid redirect chain'));
+    if (!urlStr || redirects > MAX_REDIRECTS) return reject(new Error('Redirect loop'));
     let parsed;
     try { parsed = new URL(urlStr); } catch (_) { return reject(new Error('Invalid URL')); }
+    if (Array.isArray(opts.chain)) opts.chain.push(urlStr);
     const lib = parsed.protocol === 'https:' ? https : http;
     const timeout = opts.timeout || REQ_TIMEOUT;
     const maxBytes = opts.maxBytes || MAX_BODY_BYTES;
@@ -219,7 +223,9 @@ function _request(urlStr, opts = {}, redirects = 0, jar = '') {
       // 'application/zip,...' value on binary fetches was a fingerprintable tell.
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Language': opts.lang === 'arabic' ? 'ar,en-US;q=0.8,en;q=0.6' : 'en-US,en;q=0.9',
-      'Accept-Encoding': 'gzip, deflate, br',
+      // identity on binary fetches: a zip is already compressed, and asking for
+      // br/gzip on it only adds a decode path that can stall or throw.
+      'Accept-Encoding': opts.binary ? 'identity' : 'gzip, deflate',
       'Referer': `${BASE}/`,
       'Connection': 'keep-alive',
       'Cache-Control': 'no-cache',
@@ -234,6 +240,7 @@ function _request(urlStr, opts = {}, redirects = 0, jar = '') {
       method: 'GET',
       headers,
     };
+    if (opts.insecure && parsed.protocol === 'https:') reqOpts.rejectUnauthorized = false;
     const req = lib.request(reqOpts, res => {
       const status = res.statusCode;
       const carried = _cookiesFrom(res, cookie);
@@ -275,6 +282,81 @@ function _request(urlStr, opts = {}, redirects = 0, jar = '') {
   });
 }
 
+// ================================================================
+//  HTTP/2 TRANSPORT
+// ================================================================
+// Node's https module speaks HTTP/1.1. Every real browser negotiates h2, and
+// edge WAFs fingerprint that difference. The detail page sails through on h1
+// while /download hangs, so h2 is worth a genuine attempt rather than a guess.
+function _h2request(urlStr, opts = {}, redirects = 0) {
+  return new Promise((resolve, reject) => {
+    if (!urlStr || redirects > MAX_REDIRECTS) return reject(new Error('Redirect loop'));
+    let parsed;
+    try { parsed = new URL(urlStr); } catch (_) { return reject(new Error('Invalid URL')); }
+    if (parsed.protocol !== 'https:') return reject(new Error('h2 needs https'));
+    if (Array.isArray(opts.chain)) opts.chain.push(`h2:${urlStr}`);
+    const timeout = opts.timeout || DL_HARD_TIMEOUT;
+    const maxBytes = opts.maxBytes || MAX_BODY_BYTES;
+    let settled = false;
+    let session;
+    const finish = (fn, arg) => {
+      if (settled) return;
+      settled = true;
+      try { if (session) session.destroy(); } catch (_) {}
+      fn(arg);
+    };
+    try {
+      session = http2.connect(parsed.origin, {
+        rejectUnauthorized: opts.insecure ? false : true,
+      });
+    } catch (e) { return reject(e); }
+    session.setTimeout(timeout, () => finish(reject, new Error('h2 session timeout')));
+    session.on('error', e => finish(reject, e));
+    const hdrs = {
+      ':method': 'GET',
+      ':authority': parsed.host,
+      ':scheme': 'https',
+      ':path': parsed.pathname + parsed.search,
+      'user-agent': UA,
+      'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'accept-language': 'en-US,en;q=0.9',
+      'accept-encoding': opts.binary ? 'identity' : 'gzip, deflate',
+      ...(opts.h2headers || {}),
+    };
+    let req;
+    try { req = session.request(hdrs); } catch (e) { return finish(reject, e); }
+    req.setTimeout(timeout, () => finish(reject, new Error('h2 stream timeout')));
+    let status = 0, respHeaders = {};
+    req.on('response', h => { respHeaders = h; status = Number(h[':status']) || 0; });
+    const chunks = [];
+    let total = 0;
+    req.on('data', c => {
+      total += c.length;
+      if (total > maxBytes) { req.close(http2.constants.NGHTTP2_CANCEL); return; }
+      chunks.push(c);
+    });
+    req.on('error', e => finish(reject, e));
+    req.on('end', () => {
+      let buf = Buffer.concat(chunks);
+      const loc = respHeaders['location'];
+      if (status >= 300 && status < 400 && loc) {
+        settled = true;
+        try { session.destroy(); } catch (_) {}
+        return _h2request(new URL(loc, urlStr).toString(), opts, redirects + 1).then(resolve).catch(reject);
+      }
+      if (status !== 200) return finish(resolve, { status, headers: respHeaders, body: null, cookies: '' });
+      const enc = String(respHeaders['content-encoding'] || '').toLowerCase();
+      try {
+        if (enc.includes('gzip')) buf = zlib.gunzipSync(buf);
+        else if (enc.includes('br')) buf = zlib.brotliDecompressSync(buf);
+        else if (enc.includes('deflate')) buf = zlib.inflateSync(buf);
+      } catch (e) { return finish(reject, e); }
+      finish(resolve, { status, headers: respHeaders, body: opts.binary ? buf : buf.toString('utf8'), cookies: '' });
+    });
+    req.end();
+  });
+}
+
 // Returns { body, status, cookies }. body is null when the fetch didn't yield a 200.
 async function getHTMLEx(url, tries = 3, lang = 'english', cookies = '') {
   let status = 0, jar = cookies;
@@ -295,42 +377,91 @@ async function getHTML(url, tries = 3, lang = 'english') {
   return (await getHTMLEx(url, tries, lang)).body;
 }
 
-async function getBinary(url, referer, cookies, statuses) {
-  // Two things matter here and both were missing before:
-  //  1. Referer must be the subtitle DETAIL page. A browser arrives at /download
-  //     from there, never from the site root.
-  //  2. subf2m returns transient 404/503 on /download often enough that the
-  //     reference implementation retries those specifically with a 3s sleep.
-  //     The old code retried 3x back-to-back with NO delay, which cannot clear a
-  //     throttle and looks like a bot hammering the endpoint.
-  const headers = {
+function _errLabel(e) {
+  if (!e) return 'unknown';
+  const code = e.code ? String(e.code) : '';
+  const msg = e.message ? String(e.message).slice(0, 120) : '';
+  return code ? `${code}${msg && msg !== code ? ' (' + msg + ')' : ''}` : (msg || 'unknown');
+}
+
+// Tries genuinely different transports rather than hammering the same one.
+// Each attempt records a real status or a real error code into `attempts`, so a
+// hang, a TLS failure and a redirect loop are never again flattened into -1.
+async function getBinary(url, referer, cookies, attempts) {
+  const rec = a => { if (Array.isArray(attempts)) attempts.push(a); };
+  const h1headers = {
     'Sec-Fetch-Dest': 'document',
     'Sec-Fetch-Mode': 'navigate',
     'Sec-Fetch-Site': 'same-origin',
     'Sec-Fetch-User': '?1',
   };
-  if (referer) headers.Referer = referer;
+  if (referer) h1headers.Referer = referer;
+  const h2headers = {
+    'referer': referer || `${BASE}/`,
+    'sec-fetch-dest': 'document',
+    'sec-fetch-mode': 'navigate',
+    'sec-fetch-site': 'same-origin',
+    'sec-fetch-user': '?1',
+  };
+
+  const strategies = [];
   for (let i = 0; i < DL_ATTEMPTS; i++) {
+    strategies.push({
+      via: `h1#${i + 1}`,
+      run: () => _request(url, {
+        binary: true, timeout: DL_HARD_TIMEOUT, maxBytes: MAX_SRT_BYTES,
+        headers: h1headers, cookies, chain: [],
+      }),
+    });
+  }
+  if (TRY_HTTP2) {
+    strategies.push({
+      via: 'h2',
+      run: () => _h2request(url, {
+        binary: true, timeout: DL_HARD_TIMEOUT, maxBytes: MAX_SRT_BYTES, h2headers,
+      }),
+    });
+  }
+  if (TRY_INSECURE) {
+    strategies.push({
+      via: 'h1-insecure',
+      run: () => _request(url, {
+        binary: true, timeout: DL_HARD_TIMEOUT, maxBytes: MAX_SRT_BYTES,
+        headers: h1headers, cookies, insecure: true, chain: [],
+      }),
+    });
+    if (TRY_HTTP2) strategies.push({
+      via: 'h2-insecure',
+      run: () => _h2request(url, {
+        binary: true, timeout: DL_HARD_TIMEOUT, maxBytes: MAX_SRT_BYTES, h2headers, insecure: true,
+      }),
+    });
+  }
+
+  for (let i = 0; i < strategies.length; i++) {
+    const s = strategies[i];
     try {
-      const r = await _request(url, { binary: true, timeout: DL_TIMEOUT, maxBytes: MAX_SRT_BYTES, headers, cookies });
-      if (Array.isArray(statuses)) statuses.push(r.status);
-      if (r.body !== null && r.body.length > 0) return r.body;
+      const r = await s.run();
+      rec({ via: s.via, status: r.status, bytes: r.body ? r.body.length : 0,
+            type: r.headers?.['content-type'] || null });
+      if (r.body && r.body.length > 0) return r.body;
       // Only 404/503 are worth waiting out; anything else won't change on retry.
-      if (r.status !== 404 && r.status !== 503) break;
+      if (s.via.startsWith('h1#') && r.status !== 404 && r.status !== 503) continue;
     } catch (e) {
-      if (Array.isArray(statuses)) statuses.push(-1);
+      rec({ via: s.via, error: _errLabel(e) });
     }
-    if (i < DL_ATTEMPTS - 1) await sleep(DL_RETRY_DELAY);
+    if (s.via.startsWith('h1#') && i < DL_ATTEMPTS - 1) await sleep(DL_RETRY_DELAY);
   }
+
   const enc = encodeURIComponent(url);
   for (const base of [PROXY_1, PROXY_2, PROXY_3]) {
     if (!base) continue;
     try {
       const r = await _request(`${base}${enc}`, { binary: true, timeout: DL_TIMEOUT, maxBytes: MAX_SRT_BYTES });
-      if (Array.isArray(statuses)) statuses.push(`proxy:${r.status}`);
-      if (r.body && r.body.length > 50) return r.body;
-    } catch (_) {
-      if (Array.isArray(statuses)) statuses.push('proxy:err');
+      rec({ via: `proxy ${new URL(base).hostname}`, status: r.status, bytes: r.body ? r.body.length : 0 });
+      if (r.body && r.body.length > 50 && archiveKind(r.body) !== 'html') return r.body;
+    } catch (e) {
+      rec({ via: `proxy ${new URL(base).hostname}`, error: _errLabel(e) });
     }
   }
   return null;
@@ -965,8 +1096,8 @@ function archiveKind(buf) {
 // ================================================================
 // Walks the same path a browser does: load the subtitle detail page, read the
 // real href off its download button, then request that URL carrying the detail
-// page as Referer plus any session cookie the page handed us. Every stage
-// records what happened into `trace` so /debug can show exactly where it broke.
+// page as Referer. Every stage records what happened into `trace` so /debug can
+// show exactly where it broke and why.
 async function downloadSubtitle(pageUrl, trace = {}) {
   trace.pageUrl = pageUrl;
 
@@ -1004,13 +1135,14 @@ async function downloadSubtitle(pageUrl, trace = {}) {
 
   // Stage 3 — fetch the archive, pacing the request like a real click
   await sleep(DL_STEP_DELAY);
-  const statuses = [];
-  const buf = await getBinary(downloadUrl, pageUrl, detail.cookies, statuses);
-  trace.downloadStatuses = statuses;
-  diag.lastStatuses = statuses;
+  const attempts = [];
+  const buf = await getBinary(downloadUrl, pageUrl, detail.cookies, attempts);
+  trace.attempts = attempts;
+  diag.lastStatuses = attempts;
   if (!buf) {
     diag.downloadFetchFailed++;
-    noteFail('fetch', `download returned ${statuses.join(', ') || 'nothing'}`);
+    const summary = attempts.map(a => `${a.via}=${a.error || a.status}`).join(', ');
+    noteFail('fetch', summary || 'no attempts recorded');
     trace.failedAt = 'fetch';
     return null;
   }
@@ -1091,7 +1223,7 @@ app.get('/manifest.json', (req, res) => {
     id: 'org.van.vansubf2mplus.steel',
     name: 'VanSubF2M Steel',
     description: 'Hardened Subf2m English & Arabic subtitles with release-aware matching and anti-AI filtering.',
-    version: '3.2.0',
+    version: '3.3.0',
     resources: ['subtitles'],
     types: ['movie', 'series'],
     idPrefixes: ['tt'],
@@ -1106,7 +1238,7 @@ app.get('/sub/:enc.srt', async (req, res) => {
   let pageUrl = '';
   try { pageUrl = Buffer.from(req.params.enc, 'base64url').toString('utf8'); }
   catch (_) { return res.status(400).end('Bad request'); }
-  // This now carries the subtitle DETAIL page URL (not a pre-built /download URL) —
+  // This carries the subtitle DETAIL page URL (not a pre-built /download URL) —
   // the real download link is resolved fresh from that page in fetchSRT.
   if (!/^https:\/\/subf2m\.co\/subtitles\/[^\s\/]+\/(?:english|arabic)\/\d+\/?$/i.test(pageUrl)) return res.status(403).end('Forbidden');
   const content = await fetchSRT(pageUrl);
@@ -1133,7 +1265,7 @@ if (DEBUG_ROUTE) {
     const trace = {};
     let content = null;
     try { content = await downloadSubtitle(pageUrl, trace); }
-    catch (e) { trace.threw = e?.message || String(e); }
+    catch (e) { trace.threw = _errLabel(e); }
     res.json({ ok: !!content, trace, preview: content ? content.slice(0, 300) : null });
   });
 }
@@ -1225,7 +1357,7 @@ app.get('/health', (req, res) => {
   res.json({
     ok: true,
     service: 'VanSubF2M Steel',
-    version: '3.2.0',
+    version: '3.3.0',
     debugRoute: DEBUG_ROUTE ? '/debug/<same base64url as /sub/>' : 'disabled',
     downloadDiagnostics: diag,
   });