Spaces:
Running on Zero
Running on Zero
| // Exercise production bundles first, then loaded charts with isolated API fixtures. | |
| // The fixtures replace only the inference transport and never enter the app build. | |
| const { chromium } = require('playwright'); | |
| const { build } = require('esbuild'); | |
| const { createServer } = require('node:http'); | |
| const { readFile, mkdir } = require('node:fs/promises'); | |
| const path = require('node:path'); | |
| const assert = require('node:assert/strict'); | |
| const modelId = 'g413b-lmon-exp-207-x'; | |
| const model = { model_id: modelId, num_layers: 3, layer_types: ['attention', 'attention', 'attention'] }; | |
| const distribution = { instr_t: 0.35, instr_d1: 0.05, instr_d2: 0.05, data_t: 0.3, data_d1: 0.05, prompt: 0.1, lemon: 0.1 }; | |
| const spans = Object.keys(distribution).map((category, i) => ({ | |
| id: category, category, label: category, token_start: i, token_end: i + 1, | |
| })); | |
| const kvGroups = [0, 1].map(group_index => ({ | |
| group_index, num_heads: 2, dominant_category: 'instr_t', category_distribution: distribution, | |
| })); | |
| const prefill = { | |
| model, spans, | |
| attention_by_layer: Object.fromEntries(model.layer_types.map((layer_type, layer_index) => [layer_index, | |
| layer_type !== 'attention' ? { layer_type, layer_index, metrics: null, kv_groups: null } : { | |
| layer_type, layer_index, metrics: { ifr: 0.8, das: 0.75, leak: 0.2 }, | |
| span_attention_normalized: distribution, kv_groups: kvGroups, | |
| head_focus: ['instr_t', 'instr_t', 'data_t', 'instr_d1'], head_strength: [0.8, 0.7, 0.6, 0.5], | |
| }, | |
| ])), | |
| }; | |
| const generated = { | |
| model, spans, generated_tokens: ['12', '<|open_end|>'], | |
| attention_per_step: ['12', '<|open_end|>'].map((token, step_index) => ({ | |
| token, step_index, span_attention_normalized: distribution, | |
| })), | |
| }; | |
| const views = [ | |
| { id: 1, name: 'Layer Focus', required: ['Layer heatmap', 'Attention per span', 'KV-head groups', 'Metrics'] }, | |
| { id: 2, name: 'Generation Flow', required: ['Flow view', 'Attention distribution'] }, | |
| { id: 3, name: 'Failure Diagnosis', required: ['How outcome is computed', 'Layer chart', 'Head specialization', 'Diagnosis cards'] }, | |
| ]; | |
| (async () => { | |
| const fixtureExample = JSON.parse((await readFile(path.join(__dirname, 'new_dlb_examples.jsonl'), 'utf8')).trim().split('\n')[2]); | |
| const staticRoot = path.join(__dirname, 'src/static'); | |
| const server = createServer(async (request, response) => { | |
| const pathname = new URL(request.url, 'http://local').pathname; | |
| const file = path.resolve(staticRoot, '.' + (pathname === '/' ? '/index.html' : pathname)); | |
| if (!file.startsWith(staticRoot + path.sep)) { response.writeHead(403).end(); return; } | |
| try { | |
| const mime = { '.js': 'application/javascript', '.css': 'text/css', '.html': 'text/html', '.woff': 'font/woff', '.woff2': 'font/woff2' }; | |
| response.setHeader('Content-Type', mime[path.extname(file)] || 'application/octet-stream'); | |
| response.end(await readFile(file)); | |
| } catch { response.writeHead(404).end(); } | |
| }); | |
| await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); | |
| let browser; | |
| try { | |
| browser = await chromium.launch({ headless: true, executablePath: process.env.LLMON_CHROMIUM_PATH }); | |
| const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); | |
| const errors = []; | |
| page.on('pageerror', error => errors.push(error.message)); | |
| let fixtures = false; | |
| let inferenceCalls = 0; | |
| await page.route('**/api/**', async route => { | |
| const endpoint = new URL(route.request().url()).pathname; | |
| let json; | |
| switch (endpoint) { | |
| case '/api/health': json = { model_ready: fixtures, message: 'No test model configured' }; break; | |
| case '/api/models/list': json = { | |
| active_ids: fixtures ? [modelId] : [], model_meta: { [modelId]: { label: 'Test LLMON', color: '#a56eff' } }, | |
| }; break; | |
| case '/api/data/examples': json = fixtures ? [{ id: 0, instruction: fixtureExample.instruction }] : []; break; | |
| case '/api/data/example/0': json = { ...fixtureExample, llmon_prompt: fixtureExample.llmon_prompt_dirty }; break; | |
| case '/api/attention/prefill': inferenceCalls++; json = prefill; break; | |
| case '/api/attention/generate': inferenceCalls++; json = generated; break; | |
| case '/api/benchmark/results': inferenceCalls++; json = { results: [] }; break; | |
| default: throw new Error(`Unexpected API request: ${endpoint}`); | |
| } | |
| await route.fulfill({ json }); | |
| }); | |
| const base = `http://127.0.0.1:${server.address().port}`; | |
| async function checkHelpButtons() { | |
| const names = await page.locator('button[aria-haspopup="dialog"]').evaluateAll(elements => elements.map(e => e.getAttribute('aria-label'))); | |
| assert(names.length > 0, 'Expected help buttons'); | |
| const calls = inferenceCalls; | |
| for (const name of names) { | |
| const trigger = page.getByRole('button', { name, exact: true }); | |
| await trigger.scrollIntoViewIfNeeded(); | |
| await trigger.focus(); | |
| await trigger.press('Enter'); | |
| const dialog = page.getByRole('dialog', { name, exact: true }); | |
| await dialog.waitFor(); | |
| const box = await dialog.boundingBox(); | |
| const viewport = page.viewportSize(); | |
| assert(box && box.x >= 0 && box.y >= 0 && box.x + box.width <= viewport.width + 1 && box.y + box.height <= viewport.height + 1, `Help out of viewport: ${name} ${JSON.stringify({ box, viewport })}`); | |
| assert(await dialog.evaluate(e => e.scrollWidth <= e.clientWidth + 1), `Help content overflows: ${name}`); | |
| assert((await dialog.innerText()).length > 80, `Empty help: ${name}`); | |
| await page.keyboard.press('Escape'); | |
| await dialog.waitFor({ state: 'detached' }); | |
| assert(await trigger.evaluate(e => e === document.activeElement), 'Escape should restore trigger focus'); | |
| await trigger.click(); | |
| await page.getByRole('button', { name: `Close ${name}`, exact: true }).click(); | |
| await dialog.waitFor({ state: 'detached' }); | |
| assert(await page.locator('#root').evaluate(e => e.childElementCount > 0), 'Opening help must not blank the app'); | |
| } | |
| assert.equal(inferenceCalls, calls, 'Opening help must not trigger inference'); | |
| return names; | |
| } | |
| // This fails on the shipped createPortal regression, without model access. | |
| await page.goto(base + '/experiment_1/index.html', { waitUntil: 'networkidle' }); | |
| await checkHelpButtons(); | |
| assert.deepEqual(errors, []); | |
| console.log('Production bundles: help opens without model access or a page crash.'); | |
| fixtures = true; | |
| for (const view of views) { | |
| const html = await readFile(path.join(__dirname, `frontend/experiment_${view.id}/index.html`), 'utf8'); | |
| const source = html.match(/<script type="text\/babel">([\s\S]*?)<\/script>/)[1]; | |
| const bundle = await build({ | |
| stdin: { | |
| contents: `import React from 'react'; import * as ReactDOM from 'react-dom/client'; | |
| import IBMHeader from './frontend/Header.jsx'; import InfoTip from './frontend/InfoTip.jsx'; import { parseJsonlExamples, readPalettePreference, savePalettePreference } from './frontend/data-utils.js'; | |
| import { ViewGuide, MetricsGuide, MetricSummary, ExampleContext } from './frontend/Guidance.jsx'; | |
| const fetch = window.fetch.bind(window);\n${source}`, | |
| resolveDir: __dirname, loader: 'jsx', | |
| }, | |
| bundle: true, write: false, target: 'es2020', define: { 'process.env.NODE_ENV': '"production"' }, | |
| }); | |
| await page.route(`**/experiment_${view.id}/app.js`, route => route.fulfill({ contentType: 'application/javascript', body: bundle.outputFiles[0].text })); | |
| await page.setViewportSize({ width: 1440, height: 1000 }); | |
| await page.goto(`${base}/experiment_${view.id}/index.html`, { waitUntil: 'networkidle' }); | |
| for (const name of view.required) await page.getByRole('button', { name, exact: true }).waitFor(); | |
| const guide = page.getByRole('region', { name: view.id === 2 ? 'Example context' : `${view.name} guide` }); | |
| await guide.scrollIntoViewIfNeeded(); | |
| assert(await guide.isVisible(), 'A visible introduction must be present'); | |
| assert(!/Mamba|hatched/i.test(await page.locator('body').innerText()), 'No obsolete architecture explanation'); | |
| const context = page.getByLabel('Input prompt and selected task', { exact: true }); | |
| await context.locator('summary').click(); | |
| const raw = page.getByLabel('Complete input prompt', { exact: true }); | |
| await raw.waitFor(); | |
| assert.equal(await raw.innerText(), fixtureExample.llmon_prompt_dirty); | |
| for (const marker of ['<|open|>', '<|close|>', '<|open_end|>']) assert((await raw.innerText()).includes(marker)); | |
| await context.locator('summary').click(); | |
| if (view.id === 3) { | |
| assert.equal(await page.getByText('Benchmark', { exact: true }).count(), 0); | |
| await page.getByText('Examples', { exact: true }).waitFor(); | |
| } | |
| if (view.id === 2) { | |
| const diagram = page.getByLabel('Generation flow diagram', { exact: true }); | |
| async function checkDiagramFits() { | |
| const viewport = page.viewportSize(); | |
| const bounds = await diagram.boundingBox(); | |
| assert(bounds && bounds.y >= 0 && bounds.y + bounds.height <= viewport.height + 1, | |
| `Flow does not fit: ${JSON.stringify({ viewport, bounds })}`); | |
| const clipped = await diagram.locator('svg[role=img]').evaluate(svg => { | |
| const bounds = svg.getBoundingClientRect(); | |
| return [...svg.querySelectorAll('[data-flow-category]')].some(group => { | |
| const box = group.getBoundingClientRect(); | |
| return box.top < bounds.top - 1 || box.bottom > bounds.bottom + 1 || box.right > bounds.right + 1; | |
| }); | |
| }); | |
| assert(!clipped, 'All category boxes must fit inside the flow'); | |
| assert(await diagram.locator('[data-flow-category]').evaluateAll(groups => groups.every(group => { | |
| const rect = group.querySelector('rect').getBoundingClientRect(); | |
| return [...group.querySelectorAll('text')].every(text => { | |
| const box = text.getBoundingClientRect(); | |
| return box.top >= rect.top - 1 && box.bottom <= rect.bottom + 1 && box.right <= rect.right + 1; | |
| }); | |
| })), 'Flow labels must fit inside their boxes on short screens'); | |
| } | |
| for (const [width, height] of [[1512, 780], [1440, 650], [1280, 600], [1024, 600]]) { | |
| await page.setViewportSize({ width, height }); | |
| await page.waitForTimeout(250); | |
| await checkDiagramFits(); | |
| if (process.env.LLMON_HELP_SCREENSHOTS) { | |
| await mkdir(process.env.LLMON_HELP_SCREENSHOTS, { recursive: true }); | |
| await page.screenshot({ path: path.join(process.env.LLMON_HELP_SCREENSHOTS, `flow-${width}x${height}.png`) }); | |
| } | |
| } | |
| const callsBeforeFocus = inferenceCalls; | |
| await page.getByRole('button', { name: 'Expand flow', exact: true }).click(); | |
| await page.setViewportSize({ width: 390, height: 844 }); | |
| await page.waitForTimeout(250); | |
| await checkDiagramFits(); | |
| await page.getByRole('button', { name: 'Collapse flow (Esc)', exact: true }).press('Escape'); | |
| await page.getByRole('button', { name: 'Expand flow', exact: true }).waitFor(); | |
| assert.equal(inferenceCalls, callsBeforeFocus, 'Focusing the diagram must not rerun inference'); | |
| await page.setViewportSize({ width: 1440, height: 1000 }); | |
| console.log('Feedback checks: full prompt tags, laptop diagram sizing and mobile focus view passed.'); | |
| } | |
| const names = await checkHelpButtons(); | |
| for (const name of view.required) assert(names.includes(name), `Missing help: ${name}`); | |
| const triggers = page.locator('button[aria-haspopup="dialog"]'); | |
| await triggers.first().focus(); await triggers.first().press('Enter'); | |
| await triggers.nth(1).focus(); await triggers.nth(1).press('Enter'); | |
| await page.waitForFunction(() => document.querySelectorAll('[role="dialog"]').length === 1); | |
| await page.mouse.click(page.viewportSize().width - 5, page.viewportSize().height - 5); | |
| await page.getByRole('dialog').waitFor({ state: 'detached' }); | |
| await guide.getByText('How to read this view', { exact: true }).click(); | |
| assert(await guide.locator('details').evaluate(e => e.open)); | |
| if (process.env.LLMON_HELP_SCREENSHOTS) { | |
| await mkdir(process.env.LLMON_HELP_SCREENSHOTS, { recursive: true }); | |
| await page.screenshot({ path: path.join(process.env.LLMON_HELP_SCREENSHOTS, `view-${view.id}.png`) }); | |
| } | |
| for (const width of [390, 320]) { | |
| await page.setViewportSize({ width, height: 844 }); | |
| await guide.scrollIntoViewIfNeeded(); | |
| assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1), `${view.name} overflows at ${width}px`); | |
| await checkHelpButtons(); | |
| } | |
| assert.deepEqual(errors, []); | |
| if (view.id === 1) { | |
| const customPrompt = fixtureExample.llmon_prompt_dirty.replace('5, 7', '50, 70'); | |
| await page.locator('input[type=file]').setInputFiles({ name: 'custom-prompt.jsonl', mimeType: 'application/x-ndjson', | |
| buffer: Buffer.from(JSON.stringify({ id: 0, instruction: 'Add the numbers', llmon_prompt_dirty: customPrompt })) }); | |
| await context.locator('summary').click(); | |
| await page.waitForFunction(prompt => document.querySelector('pre[aria-label="Complete input prompt"]')?.textContent === prompt, customPrompt); | |
| assert(!(await context.innerText()).includes('Expected answer: 12'), 'An uploaded prompt must not inherit a built-in reference answer'); | |
| } | |
| console.log(`${view.name}: loaded help, explanations, keyboard, dismissal, and mobile passed.`); | |
| } | |
| } finally { | |
| if (browser) await browser.close(); | |
| await new Promise(resolve => server.close(resolve)); | |
| } | |
| })().catch(error => { console.error(error); process.exitCode = 1; }); | |