src/lib/html-simplifier.js

/**
 * Pulpie WebUI - HTML Simplifier and Block Extractor
 *
 * Client-side DOM parsing and segmentation. Cleans extraneous elements,
 * resolves relative URLs to absolute URLs, tags semantic blocks with _item_id,
 * and extracts individual block HTML strings for EuroBERT classification.
 */

export const ITEM_ID_ATTR = '_item_id';

const REMOVAL_SELECTORS = [
  'script',
  'style',
  'noscript',
  'iframe',
  'frame',
  'object',
  'embed',
  'canvas',
  'svg',
  'link[rel="stylesheet"]',
  'meta',
  '[hidden]',
  '[aria-hidden="true"]',
].join(',');

const BLOCK_TAGS = new Set([
  'P',
  'H1',
  'H2',
  'H3',
  'H4',
  'H5',
  'H6',
  'BLOCKQUOTE',
  'PRE',
  'TABLE',
  'UL',
  'OL',
  'LI',
  'FIGURE',
  'ARTICLE',
  'SECTION',
  'HEADER',
  'FOOTER',
  'ASIDE',
  'NAV',
  'DIV',
]);

/**
 * Check if element has CSS rules hiding it completely.
 * @param {Element} element
 * @returns {boolean}
 */
function isElementInvisible(element) {
  const style = element.getAttribute('style') || '';
  return /display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0/i.test(style);
}

/**
 * Convert relative URLs in <img> and <a> tags to absolute URLs using baseUrl.
 * @param {Document} doc
 * @param {string} baseUrl
 */
export function resolveRelativeUrls(doc, baseUrl) {
  if (!baseUrl) return;
  try {
    const base = new URL(baseUrl);

    // Images
    const images = doc.querySelectorAll('img');
    for (const img of images) {
      const src = img.getAttribute('src');
      if (src && !src.startsWith('data:') && !src.startsWith('blob:') && !src.startsWith('javascript:')) {
        try {
          img.setAttribute('src', new URL(src, base).href);
        } catch {
          // ignore malformed URLs
        }
      }

      const dataSrc = img.getAttribute('data-src');
      if (dataSrc && !dataSrc.startsWith('data:') && !dataSrc.startsWith('blob:')) {
        try {
          const absDataSrc = new URL(dataSrc, base).href;
          img.setAttribute('data-src', absDataSrc);
          if (!img.getAttribute('src')) img.setAttribute('src', absDataSrc);
        } catch {
          // ignore
        }
      }
    }

    // Links
    const links = doc.querySelectorAll('a[href]');
    for (const a of links) {
      const href = a.getAttribute('href');
      if (href && !href.startsWith('#') && !href.startsWith('javascript:') && !href.startsWith('mailto:')) {
        try {
          a.setAttribute('href', new URL(href, base).href);
        } catch {
          // ignore
        }
      }
    }
  } catch (err) {
    console.warn('[HTMLSimplifier] Failed to parse base URL:', baseUrl, err);
  }
}

/**
 * Simplify and segment HTML into identifiable blocks.
 *
 * @param {string} rawHtml - Source HTML string
 * @param {string} [baseUrl=''] - Optional base URL for resolving relative links/images
 * @returns {{
 *   doc: Document,
 *   blocks: string[],
 *   itemIds: string[],
 *   title: string,
 *   totalNodes: number
 * }}
 */
export function simplifyAndExtractBlocks(rawHtml, baseUrl = '') {
  const parser = new DOMParser();
  const doc = parser.parseFromString(rawHtml || '', 'text/html');

  // Extract page title
  const titleElem = doc.querySelector('title');
  const title = titleElem ? titleElem.textContent.trim() : '';

  // Resolve relative URLs if baseUrl supplied
  if (baseUrl) {
    resolveRelativeUrls(doc, baseUrl);
  }

  // Remove non-content elements
  const removeTargets = doc.querySelectorAll(REMOVAL_SELECTORS);
  for (const el of removeTargets) {
    el.remove();
  }

  // Remove invisible elements
  const allElements = doc.body ? doc.body.querySelectorAll('*') : [];
  for (const el of allElements) {
    if (isElementInvisible(el)) {
      el.remove();
    }
  }

  const blocks = [];
  const itemIds = [];
  let currentId = 0;

  /**
   * Traverse nodes to identify semantic content blocks.
   * @param {Node} node
   */
  function collectBlocks(node) {
    if (!node) return;

    if (node.nodeType === Node.ELEMENT_NODE) {
      const tag = node.tagName.toUpperCase();

      // Check if this element is a standalone content block
      const isLeafContent =
        ['P', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'BLOCKQUOTE', 'PRE', 'TABLE', 'LI', 'FIGURE'].includes(tag);

      // Or an <img> tag with src
      const isImage = tag === 'IMG' && Boolean(node.getAttribute('src'));

      if (isLeafContent || isImage) {
        const textContent = node.textContent?.trim() || '';
        const hasImg = node.querySelector('img') !== null;

        // Skip empty whitespace blocks unless they contain images or tables
        if (textContent.length > 0 || hasImg || tag === 'TABLE' || isImage) {
          const itemId = String(currentId++);
          node.setAttribute(ITEM_ID_ATTR, itemId);
          blocks.push(node.outerHTML);
          itemIds.push(itemId);
          return;
        }
      }

      // If it's a container element (DIV, SECTION, ARTICLE, MAIN, BODY)
      for (const child of Array.from(node.childNodes)) {
        collectBlocks(child);
      }
    } else if (node.nodeType === Node.TEXT_NODE) {
      const text = node.textContent?.trim() || '';
      if (text.length > 10 && node.parentNode && !node.parentNode.hasAttribute(ITEM_ID_ATTR)) {
        // Wrap naked text runs
        const p = doc.createElement('p');
        p.textContent = text;
        const itemId = String(currentId++);
        p.setAttribute(ITEM_ID_ATTR, itemId);
        node.parentNode.replaceChild(p, node);
        blocks.push(p.outerHTML);
        itemIds.push(itemId);
      }
    }
  }

  if (doc.body) {
    collectBlocks(doc.body);
  }

  return {
    doc,
    blocks,
    itemIds,
    title,
    totalNodes: currentId,
  };
}