import { chromium } from 'playwright'; import fs from 'node:fs/promises'; import assert from 'node:assert/strict'; const base = process.argv[2] ?? process.env.APP_URL ?? 'http://127.0.0.1:4173/?source=local'; const label = process.argv[3] ?? process.env.TEST_LABEL ?? 'agent-browser'; const profile = process.env.BROWSER_PROFILE ?? new URL('../../validation/agent-browser-profile', import.meta.url).pathname; const output = new URL(`../../validation/${label}.json`, import.meta.url); const result = { date: new Date().toISOString(), url: base, cases: [], errors: [], requests: [] }; const context = await chromium.launchPersistentContext(profile, { executablePath: '/usr/bin/chromium', headless: true, viewport: { width: 1440, height: 1000 }, args: ['--enable-unsafe-webgpu', '--use-angle=vulkan', '--enable-features=Vulkan', '--disable-vulkan-surface'], }); context.setDefaultTimeout(240_000); try { const page = await context.newPage(); page.on('console', msg => { if (msg.type() === 'error') console.log('CONSOLE', msg.text().slice(0, 800)); }); page.on('pageerror', e => { result.errors.push(e.message); console.log('PAGEERROR', e.message); }); context.on('request', request => result.requests.push({ method: request.method(), url: request.url().split('?')[0] })); await page.goto(base); await page.waitForFunction(() => window.browserPi); await page.evaluate(() => window.browserPi.ready); console.log('INITIALIZED'); await page.screenshot({ path: '../validation/agent-browser-initial.png' }); const loadStart = Date.now(); await page.locator('#load').click(); if (process.env.TEST_CANCEL_LOAD) { await page.waitForFunction(() => document.querySelector('#progress').value > 1); await page.locator('#stop').click(); await page.waitForFunction(() => !window.browserPi.busy); assert.equal(await page.evaluate(() => window.browserPi.loaded), false); result.loadCancellation = true; await page.locator('#load').click(); } await page.waitForFunction(() => window.browserPi.loaded || (!window.browserPi.busy && document.querySelector('#notice').textContent)); assert.equal(await page.evaluate(() => window.browserPi.loaded), true, await page.locator('#load-status').textContent()); result.coldLoadMs = Date.now() - loadStart; result.device = await page.evaluate(() => window.browserPi.diagnostics.device); if (new URL(base).searchParams.has('limit128')) { assert.equal(result.device.actualLimits.maxStorageBufferBindingSize, 128 * 1024 ** 2); assert.equal(result.device.actualLimits.maxBufferSize, 256 * 1024 ** 2); } console.log('LOADED', result.coldLoadMs, result.device); // Verify the whole agent/tool/follow-up loop without any network available. await context.setOffline(true); const inferenceRequestIndex = result.requests.length; await page.evaluate(() => window.browserPi.rpc('shell', { command: 'rm -f /workspace/report.md /workspace/hello.sh /workspace/ui-test.txt /workspace/missing.txt' })); const nonce = 'LOCAL-' + crypto.randomUUID().slice(0, 8); await page.evaluate(({ nonce }) => window.browserPi.rpc('write', { path: '/workspace/secret.txt', content: nonce }), { nonce }); const tasks = [ { name: 'read-unseen-file', prompt: 'Read secret.txt with the read tool and tell me its exact contents.', check: state => { assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && !m.isError)); assert(state.messages.filter(m => m.role === 'assistant').at(-1).content.some(c => c.type === 'text' && c.text.includes(nonce))); } }, { name: 'analyze-and-write', prompt: 'Read sales.csv, calculate total revenue using bash, and save a short report to report.md. Verify the file.', check: state => { assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError)); assert.match(state.files['/workspace/report.md'] ?? '', /144/); assert(state.messages.some(m => m.role === 'toolResult' && !m.isError && m.content.some(c => c.text?.includes('144')))); } }, { name: 'write-and-run-script', prompt: 'Create a bash script called hello.sh that prints Hello from the browser, run it, and report the output.', check: state => { assert(state.files['/workspace/hello.sh']); assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError && m.content.some(c => c.text?.includes('Hello from the browser')))); } }, { name: 'edit-and-verify', prompt: 'Read hello.sh, use the edit tool to replace Hello from the browser with Hello from Pi, then run the script and report the output.', check: state => { assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'edit' && !m.isError)); assert.match(state.files['/workspace/hello.sh'], /Hello from Pi/); assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError && m.content.some(c => c.text?.includes('Hello from Pi')))); } }, { name: 'recover-tool-error', prompt: 'Read missing.txt using the read tool. If it does not exist, create it with the exact text Recovered and verify it by reading the file.', check: state => { assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && m.isError)); assert.equal(state.files['/workspace/missing.txt'].trim(), 'Recovered'); assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && !m.isError && m.content.some(c => c.text?.includes('Recovered')))); } }, ]; for (const task of tasks) { console.log('START', task.name); await page.locator('#new-chat').click(); await page.waitForFunction(() => !window.browserPi.busy); const start = Date.now(); await page.locator('.xterm-helper-textarea').focus(); await page.keyboard.type(task.prompt); console.log('TYPED', await page.evaluate(() => window.browserPi.terminal.text)); await page.waitForFunction(prompt => window.browserPi.terminal.text === prompt, task.prompt, { timeout: 15_000 }); await page.locator('#send').click(); await page.waitForFunction(() => window.browserPi.busy); await page.waitForFunction(() => !window.browserPi.busy); const state = await page.evaluate(() => window.browserPi.rpc('snapshot')); const entry = { name: task.name, elapsedMs: Date.now() - start, ...state }; result.cases.push(entry); console.log('CASE', JSON.stringify(entry)); task.check(state); } const beforeStop = await page.evaluate(() => window.browserPi.diagnostics.generations.length); const running = page.evaluate(() => window.browserPi.prompt('Write a very long detailed tutorial about shell scripting. Do not use tools.')); await page.waitForFunction(() => window.browserPi.busy); await page.waitForTimeout(900); await page.locator('.xterm-helper-textarea').focus(); await page.keyboard.press('Escape'); const stopped = await running; assert.equal(stopped.messages.filter(m => m.role === 'assistant').at(-1).stopReason, 'aborted'); result.cancellation = { passed: true, completedGenerations: await page.evaluate(() => window.browserPi.diagnostics.generations.length) - beforeStop }; result.diagnostics = await page.evaluate(() => window.browserPi.diagnostics); await page.locator('#upload').setInputFiles({ name: 'ui-test.txt', mimeType: 'text/plain', buffer: Buffer.from('Imported') }); await page.waitForFunction(() => document.querySelector('#filename').textContent === '/workspace/ui-test.txt'); await page.locator('#editor').fill('Edited through the file editor'); await page.locator('#save').click(); await page.waitForFunction(() => !window.browserPi.busy && document.querySelector('#save').disabled); await page.locator('#command').fill('cat ui-test.txt'); await page.locator('#shell-run').click(); await page.waitForFunction(() => document.querySelector('#terminal-output').textContent.includes('Edited through the file editor')); result.editorAndShell = true; result.requestsDuringOfflineTasks = result.requests.slice(inferenceRequestIndex); assert.equal(result.requestsDuringOfflineTasks.length, 0); await page.screenshot({ path: `../validation/${label}-desktop.png`, fullPage: true }); await context.setOffline(false); const beforeReload = result.requests.length; await page.reload(); await page.waitForFunction(() => window.browserPi); await page.evaluate(() => window.browserPi.ready); const restored = await page.evaluate(() => window.browserPi.rpc('snapshot')); assert.equal(restored.files['/workspace/secret.txt'], nonce); assert.equal(restored.files['/workspace/ui-test.txt'], 'Edited through the file editor'); assert(restored.files['/workspace/report.md']); assert(restored.messages.length > 0); const warmStart = Date.now(); await page.evaluate(() => window.browserPi.load()); result.warmLoadMs = Date.now() - warmStart; result.reloadWeightRequests = result.requests.slice(beforeReload).filter(r => r.url.includes('.onnx_data')); assert.equal(result.reloadWeightRequests.length, 0); await page.setViewportSize({ width: 390, height: 844 }); assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); await page.screenshot({ path: `../validation/${label}-mobile-layout.png`, fullPage: true }); result.mobileLayout = '390 × 844; desktop GPU, not a mobile hardware test'; assert.equal(result.errors.length, 0); result.passed = true; } catch (error) { result.failure = error.stack; throw error; } finally { await fs.writeFile(output, JSON.stringify(result, null, 2)); await context.close(); }