Spaces:
Sleeping
Sleeping
| #!/usr/bin/env python | |
| """Keyless multi-engine web search for the Web Research Agent (#13). | |
| No API key required — so this runs as a free, private local tool as easily as a public demo. | |
| Order: a local SearXNG (if GRITAI_SEARXNG / SEARXNG_URL is set — truest private) → Google News RSS | |
| (keyless, reliable, no rate limit) → DuckDuckGo HTML → Bing HTML. Returns structured results so the | |
| agent can fetch each URL and build numbered citations. | |
| Adapted from swarm/orchestrator/tools.py (the studio's proven keyless search) into structured output. | |
| """ | |
| import os | |
| import urllib.parse | |
| import requests | |
| import feedparser | |
| from bs4 import BeautifulSoup | |
| _UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " | |
| "(KHTML, like Gecko) Chrome/124.0 Safari/537.36") | |
| TIMEOUT = 15 | |
| def _searxng(query, n): | |
| base = os.environ.get("GRITAI_SEARXNG") or os.environ.get("SEARXNG_URL") | |
| if not base: | |
| return [] | |
| d = requests.get(f"{base}/search", params={"q": query, "format": "json"}, | |
| headers={"User-Agent": _UA}, timeout=TIMEOUT).json() | |
| return [{"title": r.get("title", ""), "url": r.get("url", ""), | |
| "snippet": (r.get("content", "") or "")[:300], "source": "searxng"} | |
| for r in d.get("results", [])[:n] if r.get("url")] | |
| def _gnews(query, n): | |
| """Google News RSS — keyless, structured, fresh, no rate limit.""" | |
| d = feedparser.parse( | |
| f"https://news.google.com/rss/search?q={urllib.parse.quote(query)}&hl=en-US&gl=US&ceid=US:en") | |
| out = [] | |
| for e in d.entries[:n]: | |
| src = e.get("source", {}).get("title", "") if isinstance(e.get("source"), dict) else "" | |
| out.append({"title": e.get("title", ""), "url": e.get("link", ""), | |
| "snippet": f"{src} · {e.get('published', '')[:16]}".strip(" ·"), "source": "google-news"}) | |
| return [r for r in out if r["url"]] | |
| def _ddg(query, n): | |
| r = requests.post("https://html.duckduckgo.com/html/", data={"q": query}, | |
| headers={"User-Agent": _UA}, timeout=TIMEOUT) | |
| soup = BeautifulSoup(r.text, "html.parser") | |
| out = [] | |
| for res in soup.select(".result")[:n]: | |
| a = res.select_one(".result__a") | |
| snip = res.select_one(".result__snippet") | |
| if a and a.get("href"): | |
| out.append({"title": a.get_text(strip=True), "url": a.get("href"), | |
| "snippet": (snip.get_text(strip=True) if snip else "")[:300], "source": "duckduckgo"}) | |
| return out | |
| def _bing(query, n): | |
| r = requests.get("https://www.bing.com/search", params={"q": query}, | |
| headers={"User-Agent": _UA}, timeout=TIMEOUT) | |
| soup = BeautifulSoup(r.text, "html.parser") | |
| out = [] | |
| for li in soup.select("li.b_algo")[:n]: | |
| a = li.select_one("h2 a") | |
| p = li.select_one(".b_caption p") or li.select_one("p") | |
| if a and a.get("href"): | |
| out.append({"title": a.get_text(strip=True), "url": a.get("href"), | |
| "snippet": (p.get_text(strip=True) if p else "")[:300], "source": "bing"}) | |
| return out | |
| def _dedup(results): | |
| seen, out = set(), [] | |
| for r in results: | |
| key = (r["url"] or "").split("#")[0].rstrip("/") | |
| if key and key not in seen: | |
| seen.add(key) | |
| out.append(r) | |
| return out | |
| def search(query: str, n: int = 6) -> list: | |
| """Return up to n structured results [{title,url,snippet,source}] with DIRECT, fetchable URLs. | |
| Order favors engines whose links can actually be read: SearXNG (local, direct) → DuckDuckGo | |
| (direct) → Bing (resolvable click-redirect). Google News is excluded here (encoded links that | |
| don't resolve to full text) — use search_news() for freshness/discovery.""" | |
| n = int(n) | |
| for engine in (_searxng, _ddg, _bing): | |
| try: | |
| hits = engine(query, n) | |
| if hits: | |
| return _dedup(hits)[:n] | |
| except Exception: | |
| continue | |
| return [] | |
| def search_news(query: str, n: int = 6) -> list: | |
| """Fresh headlines via Google News RSS — keyless, no rate limit. Links are encoded redirects | |
| (good for discovery/snippets; not reliably full-text readable), so keep this separate from the | |
| readable web search above.""" | |
| try: | |
| return _dedup(_gnews(query, int(n)))[:int(n)] | |
| except Exception: | |
| return [] | |
| if __name__ == "__main__": | |
| import sys, json | |
| q = " ".join(sys.argv[1:]) or "what is retrieval augmented generation" | |
| res = search(q, 6) | |
| print(f"query: {q}\n{len(res)} results:") | |
| print(json.dumps(res, indent=2)[:2000]) | |