File size: 9,598 Bytes
39371ea | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | import { chromium } from 'playwright';
import fs from 'node:fs/promises';
import assert from 'node:assert/strict';
const base = process.argv[2] ?? process.env.APP_URL ?? 'http://127.0.0.1:4173/?source=local';
const label = process.argv[3] ?? process.env.TEST_LABEL ?? 'agent-browser';
const profile = process.env.BROWSER_PROFILE ?? new URL('../../validation/agent-browser-profile', import.meta.url).pathname;
const output = new URL(`../../validation/${label}.json`, import.meta.url);
const result = { date: new Date().toISOString(), url: base, cases: [], errors: [], requests: [] };
const context = await chromium.launchPersistentContext(profile, {
executablePath: '/usr/bin/chromium', headless: true, viewport: { width: 1440, height: 1000 },
args: ['--enable-unsafe-webgpu', '--use-angle=vulkan', '--enable-features=Vulkan', '--disable-vulkan-surface'],
});
context.setDefaultTimeout(240_000);
try {
const page = await context.newPage();
page.on('console', msg => { if (msg.type() === 'error') console.log('CONSOLE', msg.text().slice(0, 800)); });
page.on('pageerror', e => { result.errors.push(e.message); console.log('PAGEERROR', e.message); });
context.on('request', request => result.requests.push({ method: request.method(), url: request.url().split('?')[0] }));
await page.goto(base);
await page.waitForFunction(() => window.browserPi);
await page.evaluate(() => window.browserPi.ready);
console.log('INITIALIZED');
await page.screenshot({ path: '../validation/agent-browser-initial.png' });
const loadStart = Date.now();
await page.locator('#load').click();
if (process.env.TEST_CANCEL_LOAD) {
await page.waitForFunction(() => document.querySelector('#progress').value > 1);
await page.locator('#stop').click();
await page.waitForFunction(() => !window.browserPi.busy);
assert.equal(await page.evaluate(() => window.browserPi.loaded), false);
result.loadCancellation = true;
await page.locator('#load').click();
}
await page.waitForFunction(() => window.browserPi.loaded || (!window.browserPi.busy && document.querySelector('#notice').textContent));
assert.equal(await page.evaluate(() => window.browserPi.loaded), true, await page.locator('#load-status').textContent());
result.coldLoadMs = Date.now() - loadStart;
result.device = await page.evaluate(() => window.browserPi.diagnostics.device);
if (new URL(base).searchParams.has('limit128')) {
assert.equal(result.device.actualLimits.maxStorageBufferBindingSize, 128 * 1024 ** 2);
assert.equal(result.device.actualLimits.maxBufferSize, 256 * 1024 ** 2);
}
console.log('LOADED', result.coldLoadMs, result.device);
// Verify the whole agent/tool/follow-up loop without any network available.
await context.setOffline(true);
const inferenceRequestIndex = result.requests.length;
await page.evaluate(() => window.browserPi.rpc('shell', { command: 'rm -f /workspace/report.md /workspace/hello.sh /workspace/ui-test.txt /workspace/missing.txt' }));
const nonce = 'LOCAL-' + crypto.randomUUID().slice(0, 8);
await page.evaluate(({ nonce }) => window.browserPi.rpc('write', { path: '/workspace/secret.txt', content: nonce }), { nonce });
const tasks = [
{ name: 'read-unseen-file', prompt: 'Read secret.txt with the read tool and tell me its exact contents.', check: state => {
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && !m.isError));
assert(state.messages.filter(m => m.role === 'assistant').at(-1).content.some(c => c.type === 'text' && c.text.includes(nonce)));
} },
{ name: 'analyze-and-write', prompt: 'Read sales.csv, calculate total revenue using bash, and save a short report to report.md. Verify the file.', check: state => {
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError));
assert.match(state.files['/workspace/report.md'] ?? '', /144/);
assert(state.messages.some(m => m.role === 'toolResult' && !m.isError && m.content.some(c => c.text?.includes('144'))));
} },
{ name: 'write-and-run-script', prompt: 'Create a bash script called hello.sh that prints Hello from the browser, run it, and report the output.', check: state => {
assert(state.files['/workspace/hello.sh']);
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError && m.content.some(c => c.text?.includes('Hello from the browser'))));
} },
{ name: 'edit-and-verify', prompt: 'Read hello.sh, use the edit tool to replace Hello from the browser with Hello from Pi, then run the script and report the output.', check: state => {
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'edit' && !m.isError));
assert.match(state.files['/workspace/hello.sh'], /Hello from Pi/);
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'bash' && !m.isError && m.content.some(c => c.text?.includes('Hello from Pi'))));
} },
{ name: 'recover-tool-error', prompt: 'Read missing.txt using the read tool. If it does not exist, create it with the exact text Recovered and verify it by reading the file.', check: state => {
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && m.isError));
assert.equal(state.files['/workspace/missing.txt'].trim(), 'Recovered');
assert(state.messages.some(m => m.role === 'toolResult' && m.toolName === 'read' && !m.isError && m.content.some(c => c.text?.includes('Recovered'))));
} },
];
for (const task of tasks) {
console.log('START', task.name);
await page.locator('#new-chat').click();
await page.waitForFunction(() => !window.browserPi.busy);
const start = Date.now();
await page.locator('.xterm-helper-textarea').focus();
await page.keyboard.type(task.prompt);
console.log('TYPED', await page.evaluate(() => window.browserPi.terminal.text));
await page.waitForFunction(prompt => window.browserPi.terminal.text === prompt, task.prompt, { timeout: 15_000 });
await page.locator('#send').click();
await page.waitForFunction(() => window.browserPi.busy);
await page.waitForFunction(() => !window.browserPi.busy);
const state = await page.evaluate(() => window.browserPi.rpc('snapshot'));
const entry = { name: task.name, elapsedMs: Date.now() - start, ...state };
result.cases.push(entry); console.log('CASE', JSON.stringify(entry));
task.check(state);
}
const beforeStop = await page.evaluate(() => window.browserPi.diagnostics.generations.length);
const running = page.evaluate(() => window.browserPi.prompt('Write a very long detailed tutorial about shell scripting. Do not use tools.'));
await page.waitForFunction(() => window.browserPi.busy);
await page.waitForTimeout(900);
await page.locator('.xterm-helper-textarea').focus();
await page.keyboard.press('Escape');
const stopped = await running;
assert.equal(stopped.messages.filter(m => m.role === 'assistant').at(-1).stopReason, 'aborted');
result.cancellation = { passed: true, completedGenerations: await page.evaluate(() => window.browserPi.diagnostics.generations.length) - beforeStop };
result.diagnostics = await page.evaluate(() => window.browserPi.diagnostics);
await page.locator('#upload').setInputFiles({ name: 'ui-test.txt', mimeType: 'text/plain', buffer: Buffer.from('Imported') });
await page.waitForFunction(() => document.querySelector('#filename').textContent === '/workspace/ui-test.txt');
await page.locator('#editor').fill('Edited through the file editor');
await page.locator('#save').click(); await page.waitForFunction(() => !window.browserPi.busy && document.querySelector('#save').disabled);
await page.locator('#command').fill('cat ui-test.txt'); await page.locator('#shell-run').click();
await page.waitForFunction(() => document.querySelector('#terminal-output').textContent.includes('Edited through the file editor'));
result.editorAndShell = true;
result.requestsDuringOfflineTasks = result.requests.slice(inferenceRequestIndex);
assert.equal(result.requestsDuringOfflineTasks.length, 0);
await page.screenshot({ path: `../validation/${label}-desktop.png`, fullPage: true });
await context.setOffline(false);
const beforeReload = result.requests.length;
await page.reload(); await page.waitForFunction(() => window.browserPi); await page.evaluate(() => window.browserPi.ready);
const restored = await page.evaluate(() => window.browserPi.rpc('snapshot'));
assert.equal(restored.files['/workspace/secret.txt'], nonce);
assert.equal(restored.files['/workspace/ui-test.txt'], 'Edited through the file editor');
assert(restored.files['/workspace/report.md']); assert(restored.messages.length > 0);
const warmStart = Date.now(); await page.evaluate(() => window.browserPi.load()); result.warmLoadMs = Date.now() - warmStart;
result.reloadWeightRequests = result.requests.slice(beforeReload).filter(r => r.url.includes('.onnx_data'));
assert.equal(result.reloadWeightRequests.length, 0);
await page.setViewportSize({ width: 390, height: 844 });
assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth));
await page.screenshot({ path: `../validation/${label}-mobile-layout.png`, fullPage: true });
result.mobileLayout = '390 × 844; desktop GPU, not a mobile hardware test';
assert.equal(result.errors.length, 0);
result.passed = true;
} catch (error) { result.failure = error.stack; throw error; }
finally { await fs.writeFile(output, JSON.stringify(result, null, 2)); await context.close(); }
|