Spaces:
Sleeping
Sleeping
| export interface WebSearchResult { | |
| title: string; | |
| url: string; | |
| snippet: string; | |
| source: string; | |
| } | |
| const SEARCH_TIMEOUT_MS = 5000; | |
| const MAX_RESULTS = 5; | |
| export interface RetrievedWebSource extends WebSearchResult { | |
| retrievedAt: string; | |
| kind: "extract" | "headline" | "summary"; | |
| } | |
| export async function searchWebDetailed(input: string, signal?: AbortSignal): Promise<{query:string; results:RetrievedWebSource[]; warnings:string[]}> { | |
| const query = input.replace(/^(?:can you |please |could you )?(?:search (?:on )?(?:the )?(?:internet|web)(?: about| for)?|caut[aă] (?:pe (?:net|internet) )?(?:despre )?)\s*/i, "").replace(/\s+/g," ").trim().slice(0,500); | |
| const warnings: string[] = []; | |
| if (signal?.aborted) throw new Error("Search cancelled"); | |
| let candidates: {result:WebSearchResult; kind:RetrievedWebSource["kind"]}[] = []; | |
| if (process.env.TAVILY_API_KEY) { | |
| try { candidates = (await tavilySearch(query,process.env.TAVILY_API_KEY)).map(result => ({result,kind:"extract"})); } | |
| catch { warnings.push("Search provider unavailable; trying public sources."); } | |
| } | |
| if (!candidates.length) { | |
| try { candidates = (await duckDuckGoHtmlSearch(query)).map(result => ({result,kind:"summary"})); } | |
| catch { warnings.push("General web search unavailable; trying public-source fallbacks."); } | |
| } | |
| if (!candidates.length) { | |
| const fallback = await Promise.allSettled([wikipediaSearch(query,"ro"), wikipediaSearch(query,"en"), googleNewsSearch(query), duckDuckGoInstantApi(query)]); | |
| const kinds = ["extract","extract","headline","summary"] as const; | |
| fallback.forEach((item,index) => { if (item.status === "fulfilled") candidates.push(...item.value.map(result => ({result,kind:kinds[index]}))); }); | |
| warnings.push("Public-source fallback: encyclopedia extracts, news headlines and summaries; not a full web index."); | |
| } | |
| if (signal?.aborted) throw new Error("Search cancelled"); | |
| const seen = new Set<string>(); | |
| const results: RetrievedWebSource[] = []; | |
| const terms = query.toLowerCase().split(/\s+/).filter(term => term.length > 2); | |
| const rank = (item: typeof candidates[number]) => { | |
| const title = item.result.title.toLowerCase(); | |
| const exact = title === query.toLowerCase() ? 10 : title.startsWith(query.toLowerCase() + " (") ? 7 : 0; | |
| return exact + terms.filter(term => title.includes(term)).length * 2 + (item.kind === "extract" ? 1 : 0); | |
| }; | |
| candidates.sort((a,b) => rank(b)-rank(a)); | |
| for (const {result,kind} of candidates) { | |
| try { | |
| const url = new URL(result.url); | |
| if (!['http:','https:'].includes(url.protocol) || !result.snippet.trim()) continue; | |
| url.hash = ""; | |
| const key = url.href; | |
| if (seen.has(key)) continue; | |
| seen.add(key); | |
| results.push({...result,url:key,snippet:result.snippet.slice(0,1800),kind,retrievedAt:new Date().toISOString()}); | |
| } catch { /* Discard invalid result URLs. */ } | |
| if (results.length === MAX_RESULTS) break; | |
| } | |
| await Promise.all(results.map(async result => { | |
| const url = new URL(result.url); | |
| if (!['ro.wikipedia.org','en.wikipedia.org'].includes(url.hostname) || !url.pathname.startsWith('/wiki/')) return; | |
| try { | |
| const title = decodeURIComponent(url.pathname.slice(6)).replace(/_/g,' '); | |
| const response = await fetch(`https://${url.hostname}/w/api.php?action=query&prop=extracts&explaintext=1&exintro=1&redirects=1&format=json&titles=${encodeURIComponent(title)}`, {signal:AbortSignal.timeout(SEARCH_TIMEOUT_MS)}); | |
| if (!response.ok) return; | |
| const pages = (await response.json())?.query?.pages || {}; | |
| const page = Object.values(pages)[0] as {extract?:string} | undefined; | |
| if (page?.extract) { result.snippet=page.extract.slice(0,1800); result.kind='extract'; } | |
| } catch { /* Keep the original search summary if extraction is unavailable. */ } | |
| })); | |
| return {query,results,warnings}; | |
| } | |
| function decodeHtml(value: string): string { | |
| return value | |
| .replace(/&/g, "&") | |
| .replace(/"/g, '"') | |
| .replace(/'|'/g, "'") | |
| .replace(/</g, "<") | |
| .replace(/>/g, ">") | |
| .replace(/ /g, " ") | |
| .replace(/<[^>]+>/g, " ") | |
| .replace(/\s+/g, " ") | |
| .trim(); | |
| } | |
| async function duckDuckGoHtmlSearch(query: string): Promise<WebSearchResult[]> { | |
| const response = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=ro-ro`, { | |
| signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| if (!response.ok) throw new Error("Search unavailable"); | |
| const html = await response.text(); | |
| const results: WebSearchResult[] = []; | |
| for (const block of html.split(/<h2[^>]*class="result__title"[^>]*>/).slice(1)) { | |
| const anchor = block.match(/<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/); | |
| const snippet = block.match(/<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/); | |
| if (!anchor || !snippet) continue; | |
| try { | |
| const redirect = new URL(decodeHtml(anchor[1]), "https://duckduckgo.com"); | |
| const url = new URL(redirect.searchParams.get("uddg") || redirect.href); | |
| if (!['https:','http:'].includes(url.protocol)) continue; | |
| results.push({title:decodeHtml(anchor[2]), url:url.href, snippet:decodeHtml(snippet[1]), source:url.hostname}); | |
| } catch { /* Invalid result. */ } | |
| if (results.length >= MAX_RESULTS) break; | |
| } | |
| return results; | |
| } | |
| /** | |
| * 1. Tavily Search API (High quality search when TAVILY_API_KEY is configured in .env.local) | |
| */ | |
| async function tavilySearch(query: string, apiKey: string): Promise<WebSearchResult[]> { | |
| const response = await fetch("https://api.tavily.com/search", { | |
| method: "POST", | |
| headers: { "Content-Type": "application/json" }, | |
| body: JSON.stringify({ | |
| api_key: apiKey, | |
| query, | |
| search_depth: "advanced", | |
| max_results: MAX_RESULTS, | |
| include_answer: false, | |
| }), | |
| signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| if (!response.ok) throw new Error(`Tavily search failed (${response.status})`); | |
| const data = await response.json(); | |
| return (data.results || []) | |
| .map((item: { title?: string; url?: string; content?: string }) => ({ | |
| title: decodeHtml(item.title || item.url || "Web result"), | |
| url: item.url || "", | |
| snippet: decodeHtml(item.content || ""), | |
| source: item.url ? new URL(item.url).hostname : "web", | |
| })) | |
| .filter((item: WebSearchResult) => Boolean(item.url)); | |
| } | |
| /** | |
| * 2. Wikipedia Full-Text Query API | |
| * Public encyclopedia fallback. Availability and coverage vary by query. | |
| */ | |
| async function wikipediaSearch(query: string, language?: "ro" | "en"): Promise<WebSearchResult[]> { | |
| const clean = query.replace(/[^\w\săâîșțĂÂÎȘȚ]/gi, " ").trim(); | |
| if (!clean) return []; | |
| const isRO = /[ăâîșțĂÂÎȘȚ]/.test(clean); | |
| const lang = language || (isRO ? "ro" : "en"); | |
| const searchUrl = `https://${lang}.wikipedia.org/w/api.php?action=query&list=search&srsearch=${encodeURIComponent( | |
| clean | |
| )}&format=json&utf8=1&srlimit=${MAX_RESULTS}`; | |
| const response = await fetch(searchUrl, { | |
| headers: { "User-Agent": "PurcarBot/2.0 (purcar.me knowledge search)" }, | |
| signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| if (!response.ok) return []; | |
| const data = await response.json(); | |
| const searchItems = data?.query?.search || []; | |
| if (!searchItems.length) return []; | |
| const extractResponse = await fetch(`https://${lang}.wikipedia.org/w/api.php?action=query&prop=extracts&explaintext=1&exintro=1&format=json&pageids=${searchItems.map((item: {pageid:number}) => item.pageid).join("|")}`, { | |
| headers: { "User-Agent": "PurcarBot/2.0 (purcar.me knowledge search)" }, signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| const extracts = extractResponse.ok ? (await extractResponse.json())?.query?.pages || {} : {}; | |
| return searchItems.map((item: { title: string; snippet: string; pageid: number }) => ({ | |
| title: item.title, | |
| url: `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(item.title.replace(/\s+/g, "_"))}`, | |
| snippet: decodeHtml(extracts[item.pageid]?.extract || item.snippet).slice(0, 1800), | |
| source: `${lang}.wikipedia.org`, | |
| })); | |
| } | |
| /** | |
| * 3. Google News RSS | |
| * Up-to-date live internet news and current events, works without API keys. | |
| */ | |
| async function googleNewsSearch(query: string): Promise<WebSearchResult[]> { | |
| const isRO = /[ăâîșțĂÂÎȘȚ]/.test(query) || /\b(ce|cum|de ce|este|sunt|cine|romania)\b/i.test(query); | |
| const hl = isRO ? "ro-RO" : "en-US"; | |
| const gl = isRO ? "RO" : "US"; | |
| const ceid = isRO ? "RO:ro" : "US:en"; | |
| const url = `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=${hl}&gl=${gl}&ceid=${ceid}`; | |
| const response = await fetch(url, { | |
| headers: { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", | |
| }, | |
| signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| if (!response.ok) return []; | |
| const xml = await response.text(); | |
| const results: WebSearchResult[] = []; | |
| const items = xml.match(/<item>[\s\S]*?<\/item>/gi) || []; | |
| for (const item of items.slice(0, MAX_RESULTS)) { | |
| const titleMatch = item.match(/<title>([\s\S]*?)<\/title>/i); | |
| const linkMatch = item.match(/<link>([\s\S]*?)<\/link>/i); | |
| const pubDateMatch = item.match(/<pubDate>([\s\S]*?)<\/pubDate>/i); | |
| if (titleMatch && linkMatch) { | |
| const title = decodeHtml(titleMatch[1]); | |
| const link = linkMatch[1].trim(); | |
| const pubDate = pubDateMatch ? pubDateMatch[1].trim() : ""; | |
| results.push({ | |
| title, | |
| url: link, | |
| snippet: `${title}${pubDate ? ` (${pubDate.slice(0, 16)})` : ""}`, | |
| source: "news.google.com", | |
| }); | |
| } | |
| } | |
| return results; | |
| } | |
| /** | |
| * 4. DuckDuckGo Instant Answer API | |
| */ | |
| async function duckDuckGoInstantApi(query: string): Promise<WebSearchResult[]> { | |
| const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_redirect=1&no_html=1`; | |
| const response = await fetch(url, { | |
| headers: { "User-Agent": "PurcarBot/2.0" }, | |
| signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS), | |
| }); | |
| if (!response.ok) return []; | |
| const data = await response.json(); | |
| const results: WebSearchResult[] = []; | |
| if (data.AbstractText && data.AbstractURL) { | |
| results.push({ | |
| title: data.Heading || query, | |
| url: data.AbstractURL, | |
| snippet: data.AbstractText, | |
| source: "duckduckgo.com", | |
| }); | |
| } | |
| const related = data.RelatedTopics || []; | |
| for (const topic of related) { | |
| if (topic.Text && topic.FirstURL) { | |
| results.push({ | |
| title: topic.Text.slice(0, 60), | |
| url: topic.FirstURL, | |
| snippet: topic.Text, | |
| source: "duckduckgo.com", | |
| }); | |
| } | |
| if (results.length >= MAX_RESULTS) break; | |
| } | |
| return results; | |
| } | |
| /** | |
| * Resilient Web Search aggregator: | |
| * Sequentially falls back across providers so web search functions reliably | |
| * whether running locally or hosted on any VPS / cloud datacenter. | |
| */ | |
| export async function searchWeb(query: string): Promise<WebSearchResult[]> { | |
| const cleanQuery = query.replace(/\s+/g, " ").trim().slice(0, 500); | |
| if (!cleanQuery) return []; | |
| // Tier 1: Tavily (if configured) | |
| const apiKey = process.env.TAVILY_API_KEY; | |
| if (apiKey) { | |
| try { | |
| const results = await tavilySearch(cleanQuery, apiKey); | |
| if (results.length > 0) return results; | |
| } catch (e) { | |
| console.warn("[WebSearch] Tavily failed, trying fallback:", e instanceof Error ? e.message : e); | |
| } | |
| } | |
| // Tier 2: Wikipedia Full-Text Search (100% reliable on cloud hosts) | |
| try { | |
| const wikiResults = await wikipediaSearch(cleanQuery); | |
| if (wikiResults.length > 0) return wikiResults; | |
| } catch (e) { | |
| console.warn("[WebSearch] Wikipedia query failed:", e instanceof Error ? e.message : e); | |
| } | |
| // Tier 3: Google News RSS Search (Live news & current events) | |
| try { | |
| const newsResults = await googleNewsSearch(cleanQuery); | |
| if (newsResults.length > 0) return newsResults; | |
| } catch (e) { | |
| console.warn("[WebSearch] Google News query failed:", e instanceof Error ? e.message : e); | |
| } | |
| // Tier 4: DuckDuckGo Instant Answer API | |
| try { | |
| const ddgResults = await duckDuckGoInstantApi(cleanQuery); | |
| if (ddgResults.length > 0) return ddgResults; | |
| } catch (e) { | |
| console.warn("[WebSearch] DDG Instant Answer failed:", e instanceof Error ? e.message : e); | |
| } | |
| return []; | |
| } | |