// The interface, run by Gradio as the component's `js_on_load` (in scope: element, props, watch, // trigger, upload). Gradio is the backend only: this script owns the sidebar (document, model, mode, // settings, examples) and the stage (page with boxes | text), and talks to Python through custom // events carrying plain JSON: // lo_open {path, name, longest_edge, mode} a file was uploaded: show its first page // lo_page {path, page, longest_edge, mode} show another page of the PDF // lo_run {path | example, page, model, mode, temperature, max_tokens, longest_edge} // lo_stop cancel the run // Python answers through `props.value` (see state_value() in app.py), streamed while the model // writes: {image, width, height, pages, page, path, name, mode, model, raw, markdown, blocks, // status, message, filename, example}. The model gives coordinates on a 0-1000 grid, so a box is // a
positioned in CSS percentages (x / 10 %) over the page image and follows it at any zoom. const role = name => element.querySelector(`[data-role="${name}"]`); const all = sel => element.querySelectorAll(sel); // One colour per label. Page furniture is grey on purpose; labels not listed here get the light grey. const COLORS = { title: '#d97706', section: '#b45309', text: '#2563eb', list: '#0891b2', caption: '#c026d3', footnote: '#ea580c', aside_text: '#4f46e5', header: '#78716c', footer: '#78716c', page_number: '#57534e', table: '#dc2626', chart: '#db2777', formula: '#0d9488', code: '#16a34a', image: '#7c3aed', header_image: '#8b5cf6', footer_image: '#8b5cf6', }; const colorOf = label => COLORS[label] || '#9ca3af'; const MODE_HELP = { grounding: 'Layout blocks with labels and boxes. Charts come out as tables, images as a description.', plain: 'Markdown only: the page image is the whole prompt. Figures are left out.', }; const defaults = props.defaults || {}; const S = { model: defaults.model, mode: defaults.mode || 'grounding', temperature: defaults.temperature ?? 0.2, max_tokens: defaults.max_tokens || 6144, longest_edge: defaults.longest_edge || 2048, doc: null, // {path, name, pages, page} of the open document, from the last value running: false, view: 'blocks', // blocks | doc | raw userView: false, // the reader picked the view; otherwise it follows the mode zoom: 1, // 1 = the whole page fits its pane hidden: new Set(), // labels switched off with the chips image: null, // URL shown in , to avoid resetting it on every update example: null, // index of the running example, for the highlight }; const value = () => (props.value && typeof props.value === 'object') ? props.value : {}; const grounding = v => v.mode === 'grounding'; // ---- Sidebar: build the static lists, bind the controls --------------------------------------- function buildModels() { const box = role('models'); box.replaceChildren(); (props.models || []).forEach(m => { const b = document.createElement('button'); b.type = 'button'; b.className = 'model'; b.setAttribute('role', 'radio'); b.dataset.model = m.key; b.innerHTML = ''; b.querySelector('.model-name').textContent = m.name; b.querySelector('.model-size').textContent = m.key; b.querySelector('.model-desc').textContent = m.summary; b.onclick = () => { S.model = m.key; syncControls(); }; box.appendChild(b); }); } function buildExamples() { const box = role('examples'); box.replaceChildren(); (props.examples || []).forEach((ex, i) => { const b = document.createElement('button'); b.type = 'button'; b.className = 'ex'; b.dataset.example = i; b.title = ex.caption + ' — click to run'; b.innerHTML = ''; b.querySelector('img').src = ex.thumb; b.querySelector('span').textContent = ex.caption; b.onclick = () => run({example: i}); box.appendChild(b); }); } function syncControls() { all('.model').forEach(b => b.setAttribute('aria-checked', String(b.dataset.model === S.model))); all('[data-mode]').forEach(b => b.setAttribute('aria-pressed', String(b.dataset.mode === S.mode))); role('mode-help').textContent = MODE_HELP[S.mode] || ''; all('[data-edge]').forEach(b => b.setAttribute('aria-pressed', String(Number(b.dataset.edge) === S.longest_edge))); role('temp').value = S.temperature; role('temp-out').textContent = S.temperature === 0 ? 'greedy' : S.temperature.toFixed(2); role('tokens').value = S.max_tokens; role('tokens-out').textContent = S.max_tokens.toLocaleString(); role('settings-summary').textContent = `T ${S.temperature} · ${S.max_tokens.toLocaleString()} tokens · ${S.longest_edge} px`; const run = element.querySelector('[data-act="run"]'); run.disabled = S.running || !S.doc; run.querySelector('span').textContent = S.running ? 'Running…' : S.doc ? 'Run OCR' : 'Run OCR'; element.querySelector('[data-act="stop"]').hidden = !S.running; all('.ex').forEach(b => b.classList.toggle('active', S.example !== null && Number(b.dataset.example) === S.example)); } all('[data-mode]').forEach(b => b.onclick = () => { S.mode = b.dataset.mode; syncControls(); }); all('[data-edge]').forEach(b => b.onclick = () => { S.longest_edge = Number(b.dataset.edge); syncControls(); }); role('temp').oninput = e => { S.temperature = Number(e.target.value); syncControls(); }; role('tokens').oninput = e => { S.max_tokens = Number(e.target.value); syncControls(); }; // ---- Document: upload, pages ---------------------------------------------------------------------- async function openFile(file) { if (!file) return; const ok = /\.pdf$/i.test(file.name) || (file.type || '').startsWith('image/'); if (!ok) { showMessage('Only PDF and image files are supported.', true); return; } setBusy(`Uploading ${file.name}…`); try { const {path} = await upload(file); S.example = null; trigger('lo_open', {path, name: file.name, longest_edge: S.longest_edge, mode: S.mode}); } catch (e) { console.error(e); showMessage(`Upload failed: ${e.message || e}`, true); } } function goTo(page) { if (!S.doc || !S.doc.pages) return; page = Math.min(Math.max(1, page), S.doc.pages); if (page === S.doc.page || S.running) return; trigger('lo_page', {path: S.doc.path, page, longest_edge: S.longest_edge, mode: S.mode}); } function run(extra = {}) { if (S.running) return; const payload = {model: S.model, mode: S.mode, temperature: S.temperature, max_tokens: S.max_tokens, longest_edge: S.longest_edge, ...extra}; if (extra.example === undefined) { if (!S.doc) { showMessage('Drop a PDF or an image first, or click an example.', true); return; } payload.path = S.doc.path; payload.page = S.doc.page; S.example = null; } else { S.example = extra.example; } S.running = true; syncControls(); trigger('lo_run', payload); } function stop() { trigger('lo_stop', {}); // Keep what was written so far, shown as finished (no cursor, no spinner). setValue(Object.assign({}, value(), {status: 'done', message: 'Stopped.', stopped: true})); } function setBusy(text) { role('spin').hidden = false; showMessage(text); } function showMessage(text, error = false) { const msg = role('message'); msg.textContent = text; msg.classList.toggle('error', error); } const fileInput = role('file'); role('drop-empty').onclick = () => fileInput.click(); fileInput.onchange = () => { openFile(fileInput.files[0]); fileInput.value = ''; }; // Drop a file anywhere on the stage or on the drop zone. const hasFiles = e => e.dataTransfer && [...e.dataTransfer.types].includes('Files'); let dragDepth = 0; element.addEventListener('dragenter', e => { if (hasFiles(e)) { dragDepth++; role('veil').hidden = false; role('drop-empty').classList.add('over'); } }); element.addEventListener('dragleave', e => { if (hasFiles(e) && --dragDepth <= 0) { dragDepth = 0; role('veil').hidden = true; role('drop-empty').classList.remove('over'); } }); element.addEventListener('dragover', e => { if (hasFiles(e)) e.preventDefault(); }); element.addEventListener('drop', e => { if (!hasFiles(e)) return; e.preventDefault(); dragDepth = 0; role('veil').hidden = true; role('drop-empty').classList.remove('over'); openFile(e.dataTransfer.files[0]); }); // ---- Rendering the value --------------------------------------------------------------------------- function render() { const v = value(); const blocks = grounding(v) && Array.isArray(v.blocks) ? v.blocks : []; const running = v.status === 'running'; S.running = running; S.doc = v.path ? {path: v.path, name: v.name, pages: v.pages || 1, page: v.page || 1} : null; if (v.status !== 'running' && v.status !== 'done') S.example = v.example ?? S.example; if (v.example !== undefined && v.example !== null) S.example = v.example; if (!grounding(v) && S.view === 'blocks') S.view = 'doc'; if (grounding(v) && S.view === 'doc' && !S.userView) S.view = 'blocks'; // Sidebar: the document card and the pager const has = !!S.doc; role('drop-empty').hidden = has; role('doc-card').hidden = !has; role('doc-hint').textContent = has ? '' : 'PDF or image'; if (has) { role('doc-name').textContent = S.doc.name || 'document'; role('doc-sub').textContent = [S.doc.pages > 1 ? `${S.doc.pages} pages` : null, v.width ? `${v.width}×${v.height} px` : null].filter(Boolean).join(' · '); } const pager = role('pager'); pager.hidden = !(has && S.doc.pages > 1); if (!pager.hidden) { role('page-input').value = S.doc.page; role('page-input').max = S.doc.pages; role('page-count').textContent = `/ ${S.doc.pages}`; element.querySelector('[data-act="prev"]').disabled = S.doc.page <= 1 || running; element.querySelector('[data-act="next"]').disabled = S.doc.page >= S.doc.pages || running; } syncControls(); // Toolbar role('mode-badge').textContent = v.image ? (v.mode || '') : ''; role('spin').hidden = !running; role('progress').hidden = !running; showMessage(v.message || '', v.status === 'error'); all('[data-view]').forEach(b => { b.hidden = b.dataset.view === 'blocks' && !grounding(v); b.setAttribute('aria-pressed', String(b.dataset.view === S.view)); b.disabled = !v.image; }); all('[data-zoom]').forEach(b => { b.disabled = !v.image; }); const hasText = !!(v.markdown || v.raw); all('[data-act="copy"], [data-act="download"]').forEach(b => { b.disabled = !hasText; }); renderChips(blocks); // Page pane const page = role('page'), img = role('img'); page.hidden = !v.image; role('empty').hidden = !!v.image; if (v.image && S.image !== v.image) { S.image = v.image; img.src = v.image; } if (!v.image) S.image = null; fitPage(); const boxes = role('boxes'); boxes.replaceChildren(); spy.disconnect(); blocks.forEach((block, i) => { if (S.hidden.has(block.label)) return; const [x1, y1, x2, y2] = block.bbox, label = block.label + (block.continues ? '+' : ''); const box = document.createElement('div'); box.className = 'box' + (block.continues ? ' cont' : '') + (running && i === blocks.length - 1 ? ' live' : ''); box.dataset.i = i; box.dataset.label = label; box.style.cssText = `--c:${colorOf(block.label)};left:${x1 / 10}%;top:${y1 / 10}%;width:${Math.max(0.3, (x2 - x1) / 10)}%;height:${Math.max(0.3, (y2 - y1) / 10)}%`; boxes.appendChild(box); }); // Text pane. While the model streams, keep the pane pinned to the bottom unless the reader scrolled up. const right = role('right'), panel = role('panel'); const follow = running && right.scrollTop + right.clientHeight >= right.scrollHeight - 80; panel.replaceChildren(); if (!v.image) { // nothing to show } else if (S.view === 'raw') { const pre = document.createElement('pre'); pre.className = 'raw'; pre.textContent = v.raw || (running ? '' : '(empty output)'); if (running) pre.appendChild(cursor()); panel.appendChild(pre); } else if (S.view === 'doc') { const doc = document.createElement('div'); doc.className = 'doc'; if (v.markdown) doc.appendChild(renderMarkdown(v.markdown)); else doc.innerHTML = `
${running ? 'Waiting for the model…' : v.status === 'done' ? 'Empty output' : 'Run OCR to transcribe this page'}
`; if (running && v.markdown) doc.appendChild(cursor()); panel.appendChild(doc); } else { let shown = 0; blocks.forEach((block, i) => { if (S.hidden.has(block.label)) return; shown++; const card = document.createElement('div'); card.className = 'blk'; card.dataset.i = i; card.style.setProperty('--c', colorOf(block.label)); card.innerHTML = `
${block.label}${block.continues ? '+' : ''}${block.bbox.join(', ')}
`; card.appendChild(renderMarkdown(block.text || '')); if (running && i === blocks.length - 1) card.appendChild(cursor()); panel.appendChild(card); spy.observe(card); }); if (!shown) { const note = document.createElement('div'); note.className = 'note'; note.textContent = running ? (v.raw ? 'Reading the first block…' : 'Waiting for the model…') : blocks.length ? 'Every label is hidden: click a chip to show it again' : v.status === 'done' ? 'No blocks on this page' : 'Run OCR to see the layout blocks'; panel.appendChild(note); } } if (follow) right.scrollTop = right.scrollHeight; } function cursor() { const c = document.createElement('span'); c.className = 'cursor'; return c; } // One chip per label on the page, with its count; click to hide or show the label. function renderChips(blocks) { const chips = role('chips'); chips.replaceChildren(); chips.hidden = !blocks.length; const counts = {}; blocks.forEach(block => { counts[block.label] = (counts[block.label] || 0) + 1; }); Object.entries(counts).sort((a, b) => b[1] - a[1]).forEach(([label, n]) => { const chip = document.createElement('span'); chip.className = 'chip'; chip.style.setProperty('--c', colorOf(label)); chip.setAttribute('aria-pressed', String(!S.hidden.has(label))); chip.title = 'Click to hide or show this label'; chip.innerHTML = `${label} ${n}`; chip.onclick = () => { S.hidden.has(label) ? S.hidden.delete(label) : S.hidden.add(label); render(); }; chips.appendChild(chip); }); } // ---- Markdown + LaTeX -> element. Libraries come from the component's `head`; plain text if one is missing. // marked does not parse markdown inside raw HTML, and the model writes tables as HTML with // markdown in the cells (e.g. **bold**), so bold the cell contents by hand before parsing. const boldCells = text => text.replace(/<(td|th)([^>]*)>([^<]*)<\/\1>/g, (m, tag, attrs, inner) => `<${tag}${attrs}>${inner.replace(/\*\*([^*]+)\*\*/g, '$1')}`); function renderMarkdown(text) { const el = document.createElement('div'); el.className = 'md'; let html; try { html = window.marked ? marked.parse(boldCells(text)) : null; } catch (e) { html = null; } if (html == null) { el.textContent = text; return el; } el.innerHTML = window.DOMPurify ? DOMPurify.sanitize(html, {USE_PROFILES: {html: true}}) : html; el.querySelectorAll('table').forEach(table => { // a wide table scrolls sideways inside its card const wrap = document.createElement('div'); wrap.className = 'scroll'; table.replaceWith(wrap); wrap.appendChild(table); }); try { if (window.renderMathInElement) renderMathInElement(el, { delimiters: [ {left: '$$', right: '$$', display: true}, {left: '\\[', right: '\\]', display: true}, {left: '$', right: '$', display: false}, {left: '\\(', right: '\\)', display: false}, ], throwOnError: false, }); } catch (e) {} return el; } // ---- Page fitting, zoom, pane resizing ----------------------------------------------------------------- function fitPage() { const v = value(); if (!v.image || !v.width || !v.height) return; const pane = role('left'), pad = 46; const scale = Math.min((pane.clientWidth - pad) / v.width, (pane.clientHeight - pad) / v.height); role('page').style.width = Math.max(80, Math.floor(v.width * scale * S.zoom)) + 'px'; element.querySelector('[data-zoom="fit"]').textContent = S.zoom === 1 ? 'Fit' : S.zoom.toFixed(2).replace(/\.?0+$/, '') + '×'; element.querySelector('[data-zoom="out"]').disabled = S.zoom <= 1; element.querySelector('[data-zoom="in"]').disabled = S.zoom >= 4; } function setZoom(zoom) { S.zoom = Math.min(4, Math.max(1, zoom)); fitPage(); } new ResizeObserver(() => fitPage()).observe(role('left')); role('img').addEventListener('load', fitPage); // Drag the gutter to resize the panes. const gutter = role('gutter'), panes = role('panes'); gutter.addEventListener('pointerdown', e => { e.preventDefault(); gutter.setPointerCapture(e.pointerId); gutter.classList.add('active'); const rect = panes.getBoundingClientRect(); const move = ev => { const ratio = Math.min(0.75, Math.max(0.25, (ev.clientX - rect.left) / rect.width)); panes.style.setProperty('--left', (ratio * 100).toFixed(1) + '%'); fitPage(); }; const up = () => { gutter.classList.remove('active'); gutter.removeEventListener('pointermove', move); gutter.removeEventListener('pointerup', up); }; gutter.addEventListener('pointermove', move); gutter.addEventListener('pointerup', up); }); // Boxes whose card is visible in the text pane get .inview: the page shows where you are while you scroll. const spy = new IntersectionObserver(entries => entries.forEach(entry => { const box = role('boxes').querySelector(`.box[data-i="${entry.target.dataset.i}"]`); if (box) box.classList.toggle('inview', entry.isIntersecting); }), {root: role('right'), threshold: 0.3}); // ---- Clicks on the stage, hover sync, keyboard ------------------------------------------------------- const linked = i => all(`[data-i="${i}"]`); element.addEventListener('mouseover', e => { const t = e.target.closest('[data-i]'); if (t) linked(t.dataset.i).forEach(el => el.classList.add('on')); }); element.addEventListener('mouseout', e => { const t = e.target.closest('[data-i]'); if (t) linked(t.dataset.i).forEach(el => el.classList.remove('on')); }); element.addEventListener('click', e => { const t = e.target.closest('[data-i]'); if (t) { // click one side and the other side scrolls into view if (S.view !== 'blocks' && t.classList.contains('box')) { S.view = 'blocks'; S.userView = true; render(); } linked(t.dataset.i).forEach(el => { if (el !== t) el.scrollIntoView({block: 'center', behavior: 'smooth'}); }); return; } const view = e.target.closest('[data-view]'); if (view) { S.view = view.dataset.view; S.userView = true; render(); return; } const zoom = e.target.closest('[data-zoom]'); if (zoom) { setZoom(zoom.dataset.zoom === 'in' ? S.zoom * 1.25 : zoom.dataset.zoom === 'out' ? S.zoom / 1.25 : 1); return; } const act = e.target.closest('[data-act]'); if (!act) return; switch (act.dataset.act) { case 'run': run(); break; case 'stop': stop(); break; case 'clear': clearDocument(); break; case 'prev': goTo((S.doc ? S.doc.page : 1) - 1); break; case 'next': goTo((S.doc ? S.doc.page : 1) + 1); break; case 'copy': copyMarkdown(act); break; case 'download': downloadMarkdown(); break; case 'theme': toggleTheme(); break; } }); role('page-input').onchange = e => { goTo(Number(e.target.value)); e.target.blur(); }; function clearDocument() { if (S.running) return; S.example = null; setValue({status: 'idle'}); } function markdownText() { const v = value(); return v.markdown || v.raw || ''; } async function copyMarkdown(button) { try { await navigator.clipboard.writeText(markdownText()); button.textContent = 'Copied'; button.classList.add('done'); setTimeout(() => { button.textContent = 'Copy'; button.classList.remove('done'); }, 1500); } catch (e) { console.error(e); } } function downloadMarkdown() { const v = value(); const blob = new Blob([markdownText()], {type: 'text/markdown;charset=utf-8'}); const a = document.createElement('a'); a.href = URL.createObjectURL(blob); a.download = (v.filename || 'page') + '.md'; a.click(); setTimeout(() => URL.revokeObjectURL(a.href), 1000); } document.addEventListener('keydown', e => { const typing = ['INPUT', 'SELECT', 'TEXTAREA'].includes(e.target.tagName) || e.target.isContentEditable; if ((e.metaKey || e.ctrlKey) && e.key === 'Enter') { e.preventDefault(); run(); return; } if (typing || e.metaKey || e.ctrlKey || e.altKey) return; const views = {b: 'blocks', d: 'doc', r: 'raw'}; if (views[e.key] && (e.key !== 'b' || grounding(value()))) { S.view = views[e.key]; S.userView = true; render(); } else if (e.key === '+' || e.key === '=') setZoom(S.zoom * 1.25); else if (e.key === '-') setZoom(S.zoom / 1.25); else if (e.key === '0') setZoom(1); else if (e.key === 'ArrowLeft') goTo((S.doc ? S.doc.page : 1) - 1); else if (e.key === 'ArrowRight') goTo((S.doc ? S.doc.page : 1) + 1); }); // ---- Theme: follow Gradio's `.dark` on , remembered when toggled here ----------------------------- function applyTheme(dark) { document.body.classList.toggle('dark', dark); element.querySelector('.theme .sun').style.display = dark ? 'none' : 'block'; element.querySelector('.theme .moon').style.display = dark ? 'block' : 'none'; } function toggleTheme() { const dark = !document.body.classList.contains('dark'); try { localStorage.setItem('lo-theme', dark ? 'dark' : 'light'); } catch (e) {} applyTheme(dark); } let stored = null; try { stored = localStorage.getItem('lo-theme'); } catch (e) {} applyTheme(stored ? stored === 'dark' : document.body.classList.contains('dark')); // ---- Boot -------------------------------------------------------------------------------------------- buildModels(); buildExamples(); let lastImage = null; // Finished transcriptions of this session, by page and mode, so that coming back to a page shows its // result again instead of asking for another run. The key includes the page image URL (a hash of the // rendered page), so a result is only restored for the same page at the same resolution. const results = new Map(); const resultKey = v => v.path && v.image ? `${v.path}#${v.page}#${v.mode}#${v.image}` : null; function onValue() { const v = value(); const key = resultKey(v); if (key && v.status === 'done' && !v.stopped) results.set(key, v); if (key && v.status === 'ready' && results.has(key)) { setValue(results.get(key)); return; } if (v.image !== lastImage) { lastImage = v.image; S.hidden.clear(); S.zoom = 1; S.userView = false; } render(); } // Gradio only runs watch() callbacks for values coming from the server: a value set here must be applied by hand. function setValue(v) { props.value = v; onValue(); } watch('value', onValue); render();