File size: 3,433 Bytes
3046dfd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | // Web research — search the web and fetch page content.
// Pure Node, no external dependencies, multi-OS.
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_8) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15 mona-agent/2.x';
const FETCH_TIMEOUT_MS = 15000;
const MAX_TEXT = 10000;
async function httpGet(url, headers = {}) {
const res = await fetch(url, {
headers: { 'user-agent': UA, accept: 'text/html,application/xhtml+xml', ...headers },
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
});
if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
return res.text();
}
/** Crude but effective HTML → text extraction (no external deps). */
export function htmlToText(html) {
let t = String(html || '');
t = t.replace(/<script[\s\S]*?<\/script>/gi, ' ')
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
.replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ');
t = t.replace(/<[^>]+>/g, ' ');
t = t.replace(/ /gi, ' ')
.replace(/&/gi, '&')
.replace(/</gi, '<')
.replace(/>/gi, '>')
.replace(/"/gi, '"')
.replace(/'/gi, "'");
t = t.replace(/\s+/g, ' ').trim();
return t.slice(0, MAX_TEXT);
}
/** DuckDuckGo HTML search → [{title, url, snippet}] (no API key needed). */
export async function ddgSearch(query, max = 8) {
const q = encodeURIComponent(String(query || '').slice(0, 200));
const html = await httpGet(`https://html.duckduckgo.com/html/?q=${q}`);
const results = [];
const itemRe = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
let m;
while ((m = itemRe.exec(html)) !== null && results.length < max) {
const rawUrl = m[1];
let url = rawUrl;
if (rawUrl.startsWith('//')) url = 'https:' + rawUrl;
// DuckDuckGo redirect links: extract the real target
const uddg = rawUrl.match(/uddg=([^&]+)/);
if (uddg) {
try { url = decodeURIComponent(uddg[1]); } catch { /* keep raw */ }
}
if (!url.startsWith('http')) continue;
results.push({
title: htmlToText(m[2]).slice(0, 120),
url,
snippet: htmlToText(m[3]).slice(0, 300),
});
}
return results;
}
export const web = {
name: 'web',
description: 'Web research: search the web (DuckDuckGo) and fetch page content as text.',
args: {
action: 'string — search | fetch',
query: 'string — search terms (for search)',
url: 'string — full URL (for fetch)',
max: 'number — max results (search, default 8)',
},
platform: 'any',
async run(args) {
const action = String(args.action || 'search').toLowerCase();
if (action === 'search') {
const query = String(args.query || '').trim();
if (!query) return { error: 'query required' };
const results = await ddgSearch(query, Math.min(parseInt(args.max || 8, 10) || 8, 10));
if (!results.length) return { results: [], note: 'No results returned.' };
return { results };
}
if (action === 'fetch') {
const url = String(args.url || '').trim();
if (!/^https?:\/\//i.test(url)) return { error: 'A valid http(s) URL is required' };
const html = await httpGet(url);
const text = htmlToText(html);
return { url, textLength: text.length, text: text.slice(0, MAX_TEXT) };
}
return { error: `Unknown action: ${action} (use search | fetch)` };
},
};
|