/** * Pulpie WebGPU - Main Application Controller * * Orchestrates WebGPU hardware detection, model caching, EuroBERT tokenization, * block segmentation, and rendering for sizzlebop/pulpie-orange-small-onnx. */ import { checkWebGpuSupport, createSession, runWebGpuInference } from './lib/webgpu-engine.js'; import { loadModelBuffer } from './lib/model-cache.js'; import { getTokenizer, tokenizeBlocks } from './lib/tokenizer.js'; import { simplifyAndExtractBlocks } from './lib/html-simplifier.js'; import { packChunks } from './lib/chunker.js'; import { reconstructMainContent } from './lib/reconstructor.js'; import { fetchWebpage } from './lib/url-fetcher.js'; import { SAMPLE_HTMLS } from './lib/samples.js'; import { displayResults } from './lib/ui-renderer.js'; // Application State const state = { gpuSupported: false, gpuInfo: null, isModelReady: false, isLoadingModel: false, session: null, provider: 'webgpu', // 'webgpu' | 'wasm' tokenizer: null, currentMode: 'url', // 'url' | 'html' activeOutputTab: 'preview', // 'preview' | 'markdown' | 'html' | 'blocks' lastResult: null, }; // DOM Elements const elements = { webgpuBadge: document.getElementById('webgpuBadge'), modelBanner: document.getElementById('modelBanner'), progressBarContainer: document.getElementById('progressBarContainer'), progressBarFill: document.getElementById('progressBarFill'), progressStatusText: document.getElementById('progressStatusText'), progressSpeedText: document.getElementById('progressSpeedText'), modeUrlTab: document.getElementById('modeUrlTab'), modeHtmlTab: document.getElementById('modeHtmlTab'), urlView: document.getElementById('urlView'), htmlView: document.getElementById('htmlView'), urlInput: document.getElementById('urlInput'), clearUrlBtn: document.getElementById('clearUrlBtn'), pasteUrlBtn: document.getElementById('pasteUrlBtn'), extractBtn: document.getElementById('extractBtn'), htmlInput: document.getElementById('htmlInput'), htmlDropZone: document.getElementById('htmlDropZone'), dropOverlay: document.getElementById('dropOverlay'), baseUrlInput: document.getElementById('baseUrlInput'), clearHtmlBtn: document.getElementById('clearHtmlBtn'), extractHtmlBtn: document.getElementById('extractHtmlBtn'), statusAlert: document.getElementById('statusAlert'), statusAlertIcon: document.getElementById('statusAlertIcon'), statusAlertMsg: document.getElementById('statusAlertMsg'), resultsCard: document.getElementById('resultsCard'), pageTitleDisplay: document.getElementById('pageTitleDisplay'), sourceUrlDisplay: document.getElementById('sourceUrlDisplay'), statKeptBlocks: document.getElementById('statKeptBlocks'), statRetention: document.getElementById('statRetention'), statTokens: document.getElementById('statTokens'), statTimeLabel: document.getElementById('statTimeLabel'), statGpuTime: document.getElementById('statGpuTime'), copyBtn: document.getElementById('copyBtn'), copyBtnText: document.getElementById('copyBtnText'), downloadMdBtn: document.getElementById('downloadMdBtn'), tabViewPreview: document.getElementById('tabViewPreview'), tabViewMarkdown: document.getElementById('tabViewMarkdown'), tabViewHtml: document.getElementById('tabViewHtml'), tabViewBlocks: document.getElementById('tabViewBlocks'), readerContent: document.getElementById('readerContent'), markdownContent: document.getElementById('markdownContent'), htmlContent: document.getElementById('htmlContent'), blockInspectorList: document.getElementById('blockInspectorList'), }; function showStatus(msg, type = 'info') { if (!elements.statusAlert) return; elements.statusAlert.className = `status-alert ${type}`; elements.statusAlertMsg.textContent = msg; elements.statusAlert.style.display = 'flex'; } function hideStatus() { if (elements.statusAlert) elements.statusAlert.style.display = 'none'; } async function initializeSystem() { const gpuCheck = await checkWebGpuSupport(); state.gpuSupported = gpuCheck.supported; state.gpuInfo = gpuCheck; } async function loadAndPrepareModel(providedBuffer = null) { if (state.isLoadingModel) return; state.isLoadingModel = true; hideStatus(); if (elements.progressBarContainer) { elements.progressBarContainer.style.display = 'flex'; } if (elements.progressStatusText) { elements.progressStatusText.textContent = 'Preparing model...'; } try { let modelBuffer = providedBuffer; if (!modelBuffer) { modelBuffer = await loadModelBuffer(null, ({ loaded, total, percent, speedMbps, fromCache }) => { if (elements.progressBarFill) { elements.progressBarFill.style.width = `${percent}%`; } if (fromCache) { if (elements.progressStatusText) { elements.progressStatusText.textContent = 'Loading model from browser cache...'; } if (elements.progressSpeedText) { elements.progressSpeedText.textContent = 'Cached'; } } else { const loadedMb = (loaded / (1024 * 1024)).toFixed(1); const totalMb = (total / (1024 * 1024)).toFixed(1); if (elements.progressStatusText) { elements.progressStatusText.textContent = `Downloading model: ${loadedMb} / ${totalMb} MB (${speedMbps} Mbps)`; } if (elements.progressSpeedText) { elements.progressSpeedText.textContent = `${percent}%`; } } }); } if (elements.progressStatusText) { elements.progressStatusText.textContent = 'Compiling model session...'; } if (elements.progressBarFill) { elements.progressBarFill.style.width = '100%'; } const [{ session, provider }, tokenizer] = await Promise.all([ createSession(modelBuffer, (msg) => { if (elements.progressStatusText) { elements.progressStatusText.textContent = msg; } }), getTokenizer((status) => { if (elements.progressStatusText) { elements.progressStatusText.textContent = status.status; } }), ]); state.session = session; state.provider = provider; state.tokenizer = tokenizer; state.isModelReady = true; if (elements.webgpuBadge) { if (provider === 'webgpu') { elements.webgpuBadge.textContent = 'WebGPU'; elements.webgpuBadge.className = 'badge badge-gpu'; } else { elements.webgpuBadge.textContent = 'WASM'; elements.webgpuBadge.className = 'badge badge-wasm'; } } if (elements.modelBanner) { elements.modelBanner.style.display = 'none'; } } catch (err) { console.error('[App] Model setup failed:', err); showStatus(`Failed to initialize model: ${err.message}`, 'error'); if (elements.progressStatusText) { elements.progressStatusText.textContent = 'Model loading failed.'; } } finally { state.isLoadingModel = false; } } async function handleExtract() { hideStatus(); let rawHtml = ''; let sourceUrl = ''; let baseUrl = ''; if (state.currentMode === 'url') { sourceUrl = elements.urlInput.value.trim(); if (!sourceUrl) { showStatus('Please enter a valid webpage URL to extract.', 'error'); elements.urlInput.focus(); return; } baseUrl = sourceUrl; } else { rawHtml = elements.htmlInput.value.trim(); baseUrl = elements.baseUrlInput.value.trim(); if (!rawHtml) { showStatus('Please paste HTML content or load a sample preset.', 'error'); elements.htmlInput.focus(); return; } } if (!state.isModelReady) { showStatus('Loading model into WebGPU before running extraction...', 'info'); await loadAndPrepareModel(); if (!state.isModelReady) return; } const submitBtns = [elements.extractBtn, elements.extractHtmlBtn]; submitBtns.forEach((b) => { b.disabled = true; b.querySelector('.btn-text').textContent = 'Extracting...'; }); const overallStartTime = performance.now(); try { if (state.currentMode === 'url') { showStatus('Fetching webpage content...', 'info'); const fetchResult = await fetchWebpage(sourceUrl, (msg) => showStatus(msg, 'info')); rawHtml = fetchResult.html; sourceUrl = fetchResult.finalUrl; } showStatus('Segmenting HTML into semantic content blocks...', 'info'); const { doc, blocks, itemIds, title } = simplifyAndExtractBlocks(rawHtml, baseUrl); if (blocks.length === 0) { throw new Error('No content blocks identified in HTML.'); } showStatus(`Tokenizing ${blocks.length} blocks with EuroBERT tokenizer...`, 'info'); const blockTokens = tokenizeBlocks(blocks, state.tokenizer); const maxTokens = state.provider === 'webgpu' ? 2048 : 1024; const chunks = packChunks(blockTokens, { maxTokens }); const engineName = state.provider === 'webgpu' ? 'WebGPU' : 'WASM'; showStatus(`Running ${engineName} inference across ${chunks.length} sequence chunk(s)...`, 'info'); const inferenceResult = await runWebGpuInference(state.session, chunks, itemIds, { onProgress: ({ currentChunk, totalChunks }) => { showStatus(`${engineName} forward pass: chunk ${currentChunk}/${totalChunks}...`, 'info'); }, }); showStatus('Reconstructing main content and generating Markdown...', 'info'); const { html: cleanHtml, markdown } = reconstructMainContent(doc, inferenceResult.labels); const totalTimeMs = Number((performance.now() - overallStartTime).toFixed(1)); state.lastResult = { title: title || 'Extracted Document', sourceUrl, cleanHtml, markdown, blocks, itemIds, labels: inferenceResult.labels, rawPredictions: inferenceResult.rawPredictions, inferenceTimeMs: inferenceResult.inferenceTimeMs, totalTokens: inferenceResult.totalTokens, totalTimeMs, provider: state.provider, }; displayResults(state.lastResult, elements); hideStatus(); } catch (err) { console.error('[App] Extraction failed:', err); showStatus(`Extraction failed: ${err.message}`, 'error'); } finally { submitBtns.forEach((b) => { b.disabled = false; b.querySelector('.btn-text').textContent = 'Extract'; }); } } function setupEventListeners() { elements.modeUrlTab.addEventListener('click', () => { state.currentMode = 'url'; elements.modeUrlTab.classList.add('active'); elements.modeHtmlTab.classList.remove('active'); elements.urlView.style.display = 'block'; elements.htmlView.style.display = 'none'; }); elements.modeHtmlTab.addEventListener('click', () => { state.currentMode = 'html'; elements.modeHtmlTab.classList.add('active'); elements.modeUrlTab.classList.remove('active'); elements.htmlView.style.display = 'block'; elements.urlView.style.display = 'none'; }); elements.urlInput.addEventListener('input', () => { elements.clearUrlBtn.style.display = elements.urlInput.value ? 'flex' : 'none'; }); elements.clearUrlBtn.addEventListener('click', () => { elements.urlInput.value = ''; elements.clearUrlBtn.style.display = 'none'; elements.urlInput.focus(); }); elements.pasteUrlBtn.addEventListener('click', async () => { try { const text = await navigator.clipboard.readText(); if (text) { elements.urlInput.value = text; elements.clearUrlBtn.style.display = 'flex'; } } catch { showStatus('Unable to access clipboard. Please paste manually.', 'info'); } }); elements.clearHtmlBtn.addEventListener('click', () => { elements.htmlInput.value = ''; elements.htmlInput.focus(); }); document.querySelectorAll('.preset-btn').forEach((btn) => { btn.addEventListener('click', () => { const presetKey = btn.dataset.preset; const sample = SAMPLE_HTMLS[presetKey]; if (sample) { elements.modeHtmlTab.click(); elements.htmlInput.value = sample; elements.baseUrlInput.value = 'https://usefeyn.com'; showStatus(`Loaded '${presetKey}' sample HTML document.`, 'info'); } }); }); elements.htmlDropZone.addEventListener('dragover', (e) => { e.preventDefault(); elements.dropOverlay.style.display = 'flex'; }); elements.htmlDropZone.addEventListener('dragleave', (e) => { if (!elements.htmlDropZone.contains(e.relatedTarget)) { elements.dropOverlay.style.display = 'none'; } }); elements.htmlDropZone.addEventListener('drop', (e) => { e.preventDefault(); elements.dropOverlay.style.display = 'none'; const files = e.dataTransfer?.files; if (files && files[0]) { const reader = new FileReader(); reader.onload = () => { elements.htmlInput.value = reader.result; elements.modeHtmlTab.click(); showStatus(`Loaded file: ${files[0].name}`, 'info'); }; reader.readAsText(files[0]); } }); document.querySelectorAll('.output-tab').forEach((tab) => { tab.addEventListener('click', () => { document.querySelectorAll('.output-tab').forEach((t) => t.classList.remove('active')); tab.classList.add('active'); const tabName = tab.dataset.tab; state.activeOutputTab = tabName; elements.tabViewPreview.style.display = tabName === 'preview' ? 'block' : 'none'; elements.tabViewMarkdown.style.display = tabName === 'markdown' ? 'block' : 'none'; elements.tabViewHtml.style.display = tabName === 'html' ? 'block' : 'none'; elements.tabViewBlocks.style.display = tabName === 'blocks' ? 'block' : 'none'; }); }); elements.copyBtn.addEventListener('click', async () => { if (!state.lastResult) return; const content = state.activeOutputTab === 'html' ? state.lastResult.cleanHtml : state.lastResult.markdown; try { await navigator.clipboard.writeText(content); elements.copyBtnText.textContent = 'Copied!'; setTimeout(() => (elements.copyBtnText.textContent = 'Copy'), 2000); } catch { showStatus('Clipboard copy failed. Please copy manually.', 'info'); } }); elements.downloadMdBtn.addEventListener('click', () => { if (!state.lastResult?.markdown) return; const blob = new Blob([state.lastResult.markdown], { type: 'text/markdown;charset=utf-8' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); const safeTitle = (state.lastResult.title || 'extracted') .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-|-$/g, ''); a.href = url; a.download = `${safeTitle || 'pulpie-content'}.md`; a.click(); URL.revokeObjectURL(url); }); elements.extractBtn.addEventListener('click', handleExtract); elements.extractHtmlBtn.addEventListener('click', handleExtract); window.addEventListener('keydown', (e) => { if ((e.ctrlKey || e.metaKey) && e.key === 'Enter') { e.preventDefault(); handleExtract(); } }); } document.addEventListener('DOMContentLoaded', async () => { setupEventListeners(); await initializeSystem(); loadAndPrepareModel(); });