Spaces:

Sarvesh213
/

getcert-backend

Running

getcert-backend / scripts /prerender.mjs

feat: deploy Gemma 4 optimizations and backend updates to HF Spaces

b69dd51 29 days ago

10.2 kB

/**
 * Build-time prerender: renders the public routes to static HTML inside dist/.
 *
 * Why: the app is a client-rendered SPA, so Vite's dist/index.html ships an
 * empty <div id="root">. Google can render JS, but most AI answer engines
 * (GPTBot, ClaudeBot, PerplexityBot, Applebot-Extended) do not — they read raw
 * HTML. Prerendering puts real <h1>/<p>/<a>/JSON-LD content into the file each
 * URL serves, which is what makes long-tail pages quotable.
 *
 * Route list is read from dist/sitemap.xml, which `npm run generate:sitemap`
 * already wrote before the Vite build — one source of truth, no TS imports.
 *
 * The pristine shell is held in memory and reused for every route, so writing
 * one route's output can never corrupt another's input.
 *
 * Failure policy: if Chromium cannot be launched the script warns loudly and
 * exits 0 — an un-prerendered page still works, it just loses non-JS crawlers.
 *
 *   npm run prerender
 */
import { spawnSync } from 'node:child_process';
import { createServer } from 'node:http';
import {
  createReadStream,
  existsSync,
  mkdirSync,
  readdirSync,
  readFileSync,
  statSync,
  writeFileSync,
} from 'node:fs';
import { dirname, extname, join, normalize, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = resolve(__dirname, '..');
const DIST = join(ROOT, 'dist');
const SHELL_PATH = join(DIST, 'index.html');

const MIME = {
  '.html': 'text/html; charset=utf-8',
  '.js': 'text/javascript; charset=utf-8',
  '.mjs': 'text/javascript; charset=utf-8',
  '.css': 'text/css; charset=utf-8',
  '.json': 'application/json; charset=utf-8',
  '.svg': 'image/svg+xml',
  '.png': 'image/png',
  '.jpg': 'image/jpeg',
  '.webp': 'image/webp',
  '.ico': 'image/x-icon',
  '.txt': 'text/plain; charset=utf-8',
  '.xml': 'application/xml; charset=utf-8',
  '.woff2': 'font/woff2',
};

const warn = (m) => console.warn(`[prerender] ${m}`);
const failSoft = (m) => {
  warn(m);
  warn('SKIPPING prerender — the build still succeeds, but non-JS crawlers see an empty body.');
  process.exit(0);
};

function findChromium() {
  if (process.env.PRERENDER_CHROME && existsSync(process.env.PRERENDER_CHROME)) {
    return process.env.PRERENDER_CHROME;
  }
  const roots = [
    process.env.PLAYWRIGHT_BROWSERS_PATH,
    join(process.env.HOME ?? '', '.cache/ms-playwright'),
    '/ms-playwright',
  ].filter(Boolean);

  const relatives = [
    ['chrome-linux64', 'chrome'],
    ['chrome-linux', 'chrome'],
    ['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'],
    ['chrome-win', 'chrome.exe'],
  ];

  // Distro / CI installs, where Playwright never downloaded a browser.
  const systemBins = [
    process.env.CHROME_PATH,
    '/usr/bin/google-chrome',
    '/usr/bin/google-chrome-stable',
    '/usr/bin/chromium',
    '/usr/bin/chromium-browser',
    '/opt/google/chrome/chrome',
    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
  ].filter(Boolean);
  for (const bin of systemBins) {
    if (existsSync(bin)) return bin;
  }

  for (const root of roots) {
    if (!existsSync(root)) continue;
    let dirs = [];
    try {
      dirs = readdirSync(root);
    } catch {
      continue;
    }
    const versions = dirs.filter((d) => d.startsWith('chromium-')).sort().reverse();
    for (const dir of versions) {
      for (const rel of relatives) {
        const candidate = join(root, dir, ...rel);
        if (existsSync(candidate)) return candidate;
      }
    }
  }
  return null;
}

function routesFromSitemap() {
  const sitemapPath = existsSync(join(DIST, 'sitemap.xml'))
    ? join(DIST, 'sitemap.xml')
    : join(ROOT, 'public/sitemap.xml');
  if (!existsSync(sitemapPath)) {
    failSoft('no sitemap.xml found — run `npm run generate:sitemap` first');
  }
  const xml = readFileSync(sitemapPath, 'utf8');
  const routes = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)]
    .map((m) => m[1].trim())
    .filter((loc) => loc.startsWith('https://getcert.eu.cc'))
    .map((loc) => {
      const path = loc.replace('https://getcert.eu.cc', '');
      return path.length > 1 && path.endsWith('/') ? path.slice(0, -1) : path;
    })
    .filter((p) => p.startsWith('/'));
  if (!routes.length) failSoft('sitemap.xml contained no URLs');
  return [...new Set(routes)];
}

const outputPathFor = (route) =>
  route === '/' ? SHELL_PATH : join(DIST, route.replace(/^\//, ''), 'index.html');

async function render(browser, base, route) {
  const page = await browser.newPage({
    viewport: { width: 1440, height: 1000 },
    deviceScaleFactor: 1,
    userAgent:
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  });

  // Suppress UI chrome (consent banner) so generated HTML is content only.
  await page.addInitScript(() => {
    window.__GETCERT_PRERENDER__ = true;
  });

  try {
    await page.goto(`${base}${route}`, { waitUntil: 'load', timeout: 45000 });

    // The route tree mounts after auth init, so allow a couple of attempts for
    // the <h1> to appear before calling it a failure.
    let textLength = 0;
    let title = '';
    let lastError = '';
    for (let attempt = 0; attempt < 3; attempt++) {
      await page.waitForSelector('h1', { timeout: 15000 }).catch(() => {});
      await page.waitForTimeout(1500);
      await page
        .evaluate(async () => {
          try {
            await document.fonts?.ready;
          } catch {
            /* ignore */
          }
        })
        .catch(() => {});

      textLength = await page.evaluate(() => (document.body?.innerText ?? '').trim().length);
      title = await page.title();

      if (await page.$('h1')) break;
      lastError = 'no <h1> rendered';
    }

    if (!(await page.$('h1'))) throw new Error(lastError || 'no <h1> rendered');
    if (textLength < 200) throw new Error(`body text too short (${textLength} chars)`);
    if (!title || title === 'GetCert') throw new Error(`title not route-specific: "${title}"`);

    // The <noscript> block in index.html is the fallback for when this script
    // is skipped. Now that real content is rendered, the duplicate copy would
    // be a second competing block of body text on every page.
    const html = (await page.content()).replace(/<noscript>[\s\S]*?<\/noscript>/gi, '');
    const out = outputPathFor(route);
    mkdirSync(dirname(out), { recursive: true });
    writeFileSync(out, html, 'utf8');
    return { route, status: 'ok', bytes: Buffer.byteLength(html), textLength, title };
  } finally {
    await page.close().catch(() => {});
  }
}

async function main() {
  if (!existsSync(SHELL_PATH)) failSoft(`no build at ${SHELL_PATH} — run \`vite build\` first`);

  const shell = readFileSync(SHELL_PATH, 'utf8');
  if (!shell.includes('id="root"')) failSoft('dist/index.html is not the app shell');

  let executablePath = findChromium();
  if (!executablePath && process.env.PRERENDER_INSTALL_CHROME === '1') {
    // Hosted builders (Vercel) ship no browser. Opt-in download keeps the
    // build green if it fails, and gives full prerender when it succeeds.
    warn('no Chromium found — attempting `npx playwright-core install chromium`');
    const installed = spawnSync('npx', ['playwright-core', 'install', 'chromium'], {
      cwd: ROOT,
      stdio: 'inherit',
      timeout: 300000,
    });
    if (installed.status === 0) executablePath = findChromium();
  }
  if (!executablePath) {
    failSoft('no Chromium binary — set PRERENDER_CHROME or run `npx playwright install chromium`');
  }

  let chromium;
  try {
    ({ chromium } = await import('playwright-core'));
  } catch {
    failSoft('playwright-core is not installed (npm i -D playwright-core)');
  }

  const routes = routesFromSitemap();
  const routeSet = new Set(routes);

  const server = createServer((req, res) => {
    const url = new URL(req.url ?? '/', 'http://localhost');
    let pathname = decodeURIComponent(url.pathname);
    if (pathname.length > 1 && pathname.endsWith('/')) pathname = pathname.slice(0, -1);

    const candidate = join(DIST, normalize(pathname).replace(/^(\.\.[/\\])+/, ''));
    if (
      pathname !== '/' &&
      candidate.startsWith(DIST) &&
      existsSync(candidate) &&
      statSync(candidate).isFile()
    ) {
      res.writeHead(200, { 'Content-Type': MIME[extname(candidate)] ?? 'application/octet-stream' });
      createReadStream(candidate).pipe(res);
      return;
    }

    if (routeSet.has(pathname) || !extname(pathname)) {
      res.writeHead(200, { 'Content-Type': MIME['.html'] });
      res.end(shell);
      return;
    }

    res.writeHead(404, { 'Content-Type': 'text/plain' });
    res.end('not found');
  });

  await new Promise((r) => server.listen(0, '127.0.0.1', r));
  const base = `http://127.0.0.1:${server.address().port}`;

  let browser;
  const results = [];
  try {
    browser = await chromium.launch({
      executablePath,
      headless: true,
      args: ['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
    });

    for (const route of routes) {
      const started = Date.now();
      try {
        const result = await render(browser, base, route);
        result.ms = Date.now() - started;
        results.push(result);
        console.log(
          `[prerender] ${route} OK (${result.textLength} chars, ${result.bytes} B, ${result.ms}ms)`
        );
      } catch (err) {
        const message = err?.message ?? String(err);
        results.push({ route, status: 'failed', error: message });
        warn(`${route} FAILED: ${message}`);
      }
    }
  } finally {
    await browser?.close().catch(() => {});
    server.close();
  }

  writeFileSync(
    join(DIST, '_prerender-report.json'),
    JSON.stringify({ generatedAt: new Date().toISOString(), results }, null, 2),
    'utf8'
  );

  const ok = results.filter((r) => r.status === 'ok').length;
  console.log(`[prerender] ${ok}/${results.length} routes prerendered`);
  if (ok < results.length) warn(`${results.length - ok} route(s) failed — see dist/_prerender-report.json`);
}

main().catch((err) => failSoft(`unexpected error: ${err?.stack ?? err}`));

	/**
	* Build-time prerender: renders the public routes to static HTML inside dist/.
	*
	* Why: the app is a client-rendered SPA, so Vite's dist/index.html ships an
	* empty <div id="root">. Google can render JS, but most AI answer engines
	* (GPTBot, ClaudeBot, PerplexityBot, Applebot-Extended) do not — they read raw
	* HTML. Prerendering puts real <h1>/<p>/<a>/JSON-LD content into the file each
	* URL serves, which is what makes long-tail pages quotable.
	*
	* Route list is read from dist/sitemap.xml, which `npm run generate:sitemap`
	* already wrote before the Vite build — one source of truth, no TS imports.
	*
	* The pristine shell is held in memory and reused for every route, so writing
	* one route's output can never corrupt another's input.
	*
	* Failure policy: if Chromium cannot be launched the script warns loudly and
	* exits 0 — an un-prerendered page still works, it just loses non-JS crawlers.
	*
	* npm run prerender
	*/
	import { spawnSync } from 'node:child_process';
	import { createServer } from 'node:http';
	import {
	createReadStream,
	existsSync,
	mkdirSync,
	readdirSync,
	readFileSync,
	statSync,
	writeFileSync,
	} from 'node:fs';
	import { dirname, extname, join, normalize, resolve } from 'node:path';
	import { fileURLToPath } from 'node:url';

	const __dirname = dirname(fileURLToPath(import.meta.url));
	const ROOT = resolve(__dirname, '..');
	const DIST = join(ROOT, 'dist');
	const SHELL_PATH = join(DIST, 'index.html');

	const MIME = {
	'.html': 'text/html; charset=utf-8',
	'.js': 'text/javascript; charset=utf-8',
	'.mjs': 'text/javascript; charset=utf-8',
	'.css': 'text/css; charset=utf-8',
	'.json': 'application/json; charset=utf-8',
	'.svg': 'image/svg+xml',
	'.png': 'image/png',
	'.jpg': 'image/jpeg',
	'.webp': 'image/webp',
	'.ico': 'image/x-icon',
	'.txt': 'text/plain; charset=utf-8',
	'.xml': 'application/xml; charset=utf-8',
	'.woff2': 'font/woff2',
	};

	const warn = (m) => console.warn(`[prerender] ${m}`);
	const failSoft = (m) => {
	warn(m);
	warn('SKIPPING prerender — the build still succeeds, but non-JS crawlers see an empty body.');
	process.exit(0);
	};

	function findChromium() {
	if (process.env.PRERENDER_CHROME && existsSync(process.env.PRERENDER_CHROME)) {
	return process.env.PRERENDER_CHROME;
	}
	const roots = [
	process.env.PLAYWRIGHT_BROWSERS_PATH,
	join(process.env.HOME ?? '', '.cache/ms-playwright'),
	'/ms-playwright',
	].filter(Boolean);

	const relatives = [
	['chrome-linux64', 'chrome'],
	['chrome-linux', 'chrome'],
	['chrome-mac', 'Chromium.app', 'Contents', 'MacOS', 'Chromium'],
	['chrome-win', 'chrome.exe'],
	];

	// Distro / CI installs, where Playwright never downloaded a browser.
	const systemBins = [
	process.env.CHROME_PATH,
	'/usr/bin/google-chrome',
	'/usr/bin/google-chrome-stable',
	'/usr/bin/chromium',
	'/usr/bin/chromium-browser',
	'/opt/google/chrome/chrome',
	'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
	].filter(Boolean);
	for (const bin of systemBins) {
	if (existsSync(bin)) return bin;
	}

	for (const root of roots) {
	if (!existsSync(root)) continue;
	let dirs = [];
	try {
	dirs = readdirSync(root);
	} catch {
	continue;
	}
	const versions = dirs.filter((d) => d.startsWith('chromium-')).sort().reverse();
	for (const dir of versions) {
	for (const rel of relatives) {
	const candidate = join(root, dir, ...rel);
	if (existsSync(candidate)) return candidate;
	}
	}
	}
	return null;
	}

	function routesFromSitemap() {
	const sitemapPath = existsSync(join(DIST, 'sitemap.xml'))
	? join(DIST, 'sitemap.xml')
	: join(ROOT, 'public/sitemap.xml');
	if (!existsSync(sitemapPath)) {
	failSoft('no sitemap.xml found — run `npm run generate:sitemap` first');
	}
	const xml = readFileSync(sitemapPath, 'utf8');
	const routes = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)]
	.map((m) => m[1].trim())
	.filter((loc) => loc.startsWith('https://getcert.eu.cc'))
	.map((loc) => {
	const path = loc.replace('https://getcert.eu.cc', '');
	return path.length > 1 && path.endsWith('/') ? path.slice(0, -1) : path;
	})
	.filter((p) => p.startsWith('/'));
	if (!routes.length) failSoft('sitemap.xml contained no URLs');
	return [...new Set(routes)];
	}

	const outputPathFor = (route) =>
	route === '/' ? SHELL_PATH : join(DIST, route.replace(/^\//, ''), 'index.html');

	async function render(browser, base, route) {
	const page = await browser.newPage({
	viewport: { width: 1440, height: 1000 },
	deviceScaleFactor: 1,
	userAgent:
	'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
	});

	// Suppress UI chrome (consent banner) so generated HTML is content only.
	await page.addInitScript(() => {
	window.__GETCERT_PRERENDER__ = true;
	});

	try {
	await page.goto(`${base}${route}`, { waitUntil: 'load', timeout: 45000 });

	// The route tree mounts after auth init, so allow a couple of attempts for
	// the <h1> to appear before calling it a failure.
	let textLength = 0;
	let title = '';
	let lastError = '';
	for (let attempt = 0; attempt < 3; attempt++) {
	await page.waitForSelector('h1', { timeout: 15000 }).catch(() => {});
	await page.waitForTimeout(1500);
	await page
	.evaluate(async () => {
	try {
	await document.fonts?.ready;
	} catch {
	/* ignore */
	}
	})
	.catch(() => {});

	textLength = await page.evaluate(() => (document.body?.innerText ?? '').trim().length);
	title = await page.title();

	if (await page.$('h1')) break;
	lastError = 'no <h1> rendered';
	}

	if (!(await page.$('h1'))) throw new Error(lastError \|\| 'no <h1> rendered');
	if (textLength < 200) throw new Error(`body text too short (${textLength} chars)`);
	if (!title \|\| title === 'GetCert') throw new Error(`title not route-specific: "${title}"`);

	// The <noscript> block in index.html is the fallback for when this script
	// is skipped. Now that real content is rendered, the duplicate copy would
	// be a second competing block of body text on every page.
	const html = (await page.content()).replace(/<noscript>[\s\S]*?<\/noscript>/gi, '');
	const out = outputPathFor(route);
	mkdirSync(dirname(out), { recursive: true });
	writeFileSync(out, html, 'utf8');
	return { route, status: 'ok', bytes: Buffer.byteLength(html), textLength, title };
	} finally {
	await page.close().catch(() => {});
	}
	}

	async function main() {
	if (!existsSync(SHELL_PATH)) failSoft(`no build at ${SHELL_PATH} — run \`vite build\` first`);

	const shell = readFileSync(SHELL_PATH, 'utf8');
	if (!shell.includes('id="root"')) failSoft('dist/index.html is not the app shell');

	let executablePath = findChromium();
	if (!executablePath && process.env.PRERENDER_INSTALL_CHROME === '1') {
	// Hosted builders (Vercel) ship no browser. Opt-in download keeps the
	// build green if it fails, and gives full prerender when it succeeds.
	warn('no Chromium found — attempting `npx playwright-core install chromium`');
	const installed = spawnSync('npx', ['playwright-core', 'install', 'chromium'], {
	cwd: ROOT,
	stdio: 'inherit',
	timeout: 300000,
	});
	if (installed.status === 0) executablePath = findChromium();
	}
	if (!executablePath) {
	failSoft('no Chromium binary — set PRERENDER_CHROME or run `npx playwright install chromium`');
	}

	let chromium;
	try {
	({ chromium } = await import('playwright-core'));
	} catch {
	failSoft('playwright-core is not installed (npm i -D playwright-core)');
	}

	const routes = routesFromSitemap();
	const routeSet = new Set(routes);

	const server = createServer((req, res) => {
	const url = new URL(req.url ?? '/', 'http://localhost');
	let pathname = decodeURIComponent(url.pathname);
	if (pathname.length > 1 && pathname.endsWith('/')) pathname = pathname.slice(0, -1);

	const candidate = join(DIST, normalize(pathname).replace(/^(\.\.[/\\])+/, ''));
	if (
	pathname !== '/' &&
	candidate.startsWith(DIST) &&
	existsSync(candidate) &&
	statSync(candidate).isFile()
	) {
	res.writeHead(200, { 'Content-Type': MIME[extname(candidate)] ?? 'application/octet-stream' });
	createReadStream(candidate).pipe(res);
	return;
	}

	if (routeSet.has(pathname) \|\| !extname(pathname)) {
	res.writeHead(200, { 'Content-Type': MIME['.html'] });
	res.end(shell);
	return;
	}

	res.writeHead(404, { 'Content-Type': 'text/plain' });
	res.end('not found');
	});

	await new Promise((r) => server.listen(0, '127.0.0.1', r));
	const base = `http://127.0.0.1:${server.address().port}`;

	let browser;
	const results = [];
	try {
	browser = await chromium.launch({
	executablePath,
	headless: true,
	args: ['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
	});

	for (const route of routes) {
	const started = Date.now();
	try {
	const result = await render(browser, base, route);
	result.ms = Date.now() - started;
	results.push(result);
	console.log(
	`[prerender] ${route} OK (${result.textLength} chars, ${result.bytes} B, ${result.ms}ms)`
	);
	} catch (err) {
	const message = err?.message ?? String(err);
	results.push({ route, status: 'failed', error: message });
	warn(`${route} FAILED: ${message}`);
	}
	}
	} finally {
	await browser?.close().catch(() => {});
	server.close();
	}

	writeFileSync(
	join(DIST, '_prerender-report.json'),
	JSON.stringify({ generatedAt: new Date().toISOString(), results }, null, 2),
	'utf8'
	);

	const ok = results.filter((r) => r.status === 'ok').length;
	console.log(`[prerender] ${ok}/${results.length} routes prerendered`);
	if (ok < results.length) warn(`${results.length - ok} route(s) failed — see dist/_prerender-report.json`);
	}

	main().catch((err) => failSoft(`unexpected error: ${err?.stack ?? err}`));