| import { chromium } from 'playwright'; |
| import fs from 'node:fs/promises'; |
|
|
| const root = new URL('../..', import.meta.url).pathname; |
| const url = process.argv[2] ?? 'https://mike0021-minicpm5-2b-webgpu-pi-http.static.hf.space/'; |
| const result = { date: new Date().toISOString(), url, cases: [], errors: [], requests: [] }; |
| const context = await chromium.launchPersistentContext(root + '/validation/http-browser-profile', { |
| executablePath:'/usr/bin/chromium', headless:true, viewport:{width:1440,height:1000}, |
| args:['--enable-unsafe-webgpu','--use-angle=vulkan','--enable-features=Vulkan','--disable-vulkan-surface'], |
| }); |
| context.setDefaultTimeout(300_000); |
| const page = await context.newPage(); |
| page.on('pageerror', e => result.errors.push(e.message)); |
| context.on('request', r => { |
| const address = new URL(r.url()); |
| if (/hf\.co$/.test(address.hostname) || address.hostname.includes('xethub')) address.search = ''; |
| result.requests.push({url:address.href,method:r.method()}); |
| }); |
| const rpc = (action,args) => page.evaluate(({action,args})=>window.browserPi.rpc(action,args),{action,args}); |
| const save = () => fs.writeFile(root+'/validation/http-agent-live.json',JSON.stringify(result,null,2)+'\n'); |
| const tasks = [ |
| { name:'fetch-and-answer', prompt:'Fetch https://jsonplaceholder.typicode.com/todos/1 and tell me its title and whether it is completed.', check: c => c.httpToolSuccess && /delectus aut autem/.test(c.final) && /false|not complet|incomplete/i.test(c.final) }, |
| { name:'fetch-count-write', prompt:'Download https://jsonplaceholder.typicode.com/users into users.json. Count the users and save a report containing that count to users-report.txt. Verify the report.', check: c => c.httpToolSuccess && JSON.parse(c.state.files['/workspace/users.json']).length === 10 && /\b10\b/.test(c.state.files['/workspace/users-report.txt']??'') && c.toolResults.some(t => !t.isError && /10/.test(JSON.stringify(t.content)) && t.toolName === 'read') }, |
| { name:'github-api', prompt:'Use the GitHub API to tell me the full repository name and license of huggingface/transformers.js.', check: c => c.httpToolSuccess && /huggingface\/transformers\.js/.test(c.final) && /Apache[- ]2\.0/i.test(c.final) }, |
| { name:'post-json', prompt:'POST the JSON {"demo":"browser-http","value":144} to https://httpbin.org/post. Tell me the JSON it echoes back.', check: c => c.httpToolSuccess && /browser-http/.test(c.final) && /144/.test(c.final) && c.requests.filter(r=>r.method==='POST'&&r.url==='https://httpbin.org/post').length === 1 }, |
| { name:'http-rejected', prompt:'Fetch http://example.com/ exactly as written, without switching protocols, and tell me its page title.', check: c => /HTTPS|protocol/i.test(c.final) && !/Example Domain/.test(c.final) && !c.requests.some(r=>new URL(r.url).hostname==='example.com') }, |
| { name:'cors-error', prompt:'Fetch https://huggingface.co/ and tell me its page title.', check: c => /CORS|browser.*block|block.*browser|request failed|unable.*fetch|cannot.*fetch|can.t.*fetch/i.test(c.final) && c.toolResults.some(t => t.isError && /CORS/.test(JSON.stringify(t.content))) && !/bot detection|blocking automated/i.test(c.final) && c.requests.filter(r=>r.url==='https://huggingface.co/').length <= 2 }, |
| { name:'fetch-and-answer-repeat', prompt:'Fetch https://jsonplaceholder.typicode.com/todos/1 and tell me its title and whether it is completed.', check: c => c.httpToolSuccess && /delectus aut autem/.test(c.final) && /false|not complet|incomplete/i.test(c.final) }, |
| ]; |
| let monitor; |
| try { |
| await page.goto(url); await page.waitForFunction(()=>window.browserPi?.terminal); await page.evaluate(()=>window.browserPi.ready); |
| result.wasCached=await rpc('cache_status'); console.log('MODEL_CACHE',result.wasCached); |
| const start=Date.now(); |
| monitor=setInterval(async()=>{try{console.log('LOAD_PROGRESS',Math.round((Date.now()-start)/1000),await page.locator('#load-status').textContent(),await page.locator('#load-detail').textContent());}catch{}},20000); |
| await page.evaluate(()=>window.browserPi.load()); clearInterval(monitor); |
| result.modelLoadMs=Date.now()-start; result.device=await page.evaluate(()=>window.browserPi.diagnostics.device); |
| console.log('MODEL_LOADED',result.modelLoadMs,JSON.stringify(result.device)); await save(); |
| for (const task of tasks) { |
| await page.locator('#new-chat').click(); await page.waitForFunction(()=>!window.browserPi.busy); |
| const start=Date.now(), requestStart=result.requests.length; |
| console.log('START',task.name); |
| let count=0; |
| monitor=setInterval(async()=>{try{const state=await rpc('snapshot');const tools=state.messages.filter(m=>m.role==='toolResult');console.log('PROGRESS',task.name,Math.round((Date.now()-start)/1000),'tools',tools.length);for(const t of tools.slice(count))console.log('TOOL',t.toolName,t.isError,JSON.stringify(t.content).slice(0,400));count=tools.length;}catch{}},15000); |
| const pending=page.evaluate(text=>window.browserPi.prompt(text),task.prompt); |
| let timer; |
| try { await Promise.race([pending,new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error('Trial exceeded 180 seconds')),180000);})]); } |
| catch(e){ await rpc('stop'); await pending.catch(()=>{}); task.failure=e.message; } |
| finally{clearTimeout(timer);clearInterval(monitor);} |
| const state=await rpc('snapshot'); |
| const entry={name:task.name,prompt:task.prompt,elapsedMs:Date.now()-start,state,requests:result.requests.slice(requestStart),failure:task.failure}; |
| entry.final=state.messages.filter(m=>m.role==='assistant').at(-1)?.content.filter(c=>c.type==='text').map(c=>c.text).join('\n')??''; |
| entry.toolResults=state.messages.filter(m=>m.role==='toolResult'); |
| entry.calls=state.messages.filter(m=>m.role==='assistant').flatMap(m=>m.content.filter(c=>c.type==='toolCall')); |
| entry.httpToolSuccess=entry.toolResults.some(t=>t.toolName==='bash'&&!t.isError&&entry.calls.some(c=>c.id===t.toolCallId&&/\bcurl\b|\bwget\b/.test(c.arguments?.command??''))); |
| try{entry.passed=!entry.failure&&task.check(entry);}catch(e){entry.passed=false;entry.checkError=e.message;} |
| result.cases.push(entry);await save(); |
| console.log('RESULT',JSON.stringify({name:entry.name,elapsedMs:entry.elapsedMs,passed:entry.passed,final:entry.final,calls:entry.calls})); |
| await page.screenshot({path:root+'/validation/http-agent-'+task.name+'.png',fullPage:true}); |
| } |
| result.passed=result.cases.every(c=>c.passed)&&!result.errors.length; |
| result.diagnostics=await page.evaluate(()=>window.browserPi.diagnostics); |
| console.log('SUMMARY',JSON.stringify({passed:result.passed,cases:result.cases.map(c=>({name:c.name,passed:c.passed,seconds:c.elapsedMs/1000}))})); |
| }catch(e){result.failure=e.stack;console.log('FAIL',e.stack);process.exitCode=1;} |
| finally{clearInterval(monitor);await save();await context.close();} |
|
|
