#!/usr/bin/env python
"""Keyless multi-engine web search for the Web Research Agent (#13).

No API key required — so this runs as a free, private local tool as easily as a public demo.
Order: a local SearXNG (if GRITAI_SEARXNG / SEARXNG_URL is set — truest private) → Google News RSS
(keyless, reliable, no rate limit) → DuckDuckGo HTML → Bing HTML. Returns structured results so the
agent can fetch each URL and build numbered citations.

Adapted from swarm/orchestrator/tools.py (the studio's proven keyless search) into structured output.
"""
import os
import urllib.parse

import requests
import feedparser
from bs4 import BeautifulSoup

_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
       "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
TIMEOUT = 15


def _searxng(query, n):
    base = os.environ.get("GRITAI_SEARXNG") or os.environ.get("SEARXNG_URL")
    if not base:
        return []
    d = requests.get(f"{base}/search", params={"q": query, "format": "json"},
                     headers={"User-Agent": _UA}, timeout=TIMEOUT).json()
    return [{"title": r.get("title", ""), "url": r.get("url", ""),
             "snippet": (r.get("content", "") or "")[:300], "source": "searxng"}
            for r in d.get("results", [])[:n] if r.get("url")]


def _gnews(query, n):
    """Google News RSS — keyless, structured, fresh, no rate limit."""
    d = feedparser.parse(
        f"https://news.google.com/rss/search?q={urllib.parse.quote(query)}&hl=en-US&gl=US&ceid=US:en")
    out = []
    for e in d.entries[:n]:
        src = e.get("source", {}).get("title", "") if isinstance(e.get("source"), dict) else ""
        out.append({"title": e.get("title", ""), "url": e.get("link", ""),
                    "snippet": f"{src} · {e.get('published', '')[:16]}".strip(" ·"), "source": "google-news"})
    return [r for r in out if r["url"]]


def _ddg(query, n):
    r = requests.post("https://html.duckduckgo.com/html/", data={"q": query},
                      headers={"User-Agent": _UA}, timeout=TIMEOUT)
    soup = BeautifulSoup(r.text, "html.parser")
    out = []
    for res in soup.select(".result")[:n]:
        a = res.select_one(".result__a")
        snip = res.select_one(".result__snippet")
        if a and a.get("href"):
            out.append({"title": a.get_text(strip=True), "url": a.get("href"),
                        "snippet": (snip.get_text(strip=True) if snip else "")[:300], "source": "duckduckgo"})
    return out


def _bing(query, n):
    r = requests.get("https://www.bing.com/search", params={"q": query},
                     headers={"User-Agent": _UA}, timeout=TIMEOUT)
    soup = BeautifulSoup(r.text, "html.parser")
    out = []
    for li in soup.select("li.b_algo")[:n]:
        a = li.select_one("h2 a")
        p = li.select_one(".b_caption p") or li.select_one("p")
        if a and a.get("href"):
            out.append({"title": a.get_text(strip=True), "url": a.get("href"),
                        "snippet": (p.get_text(strip=True) if p else "")[:300], "source": "bing"})
    return out


def _dedup(results):
    seen, out = set(), []
    for r in results:
        key = (r["url"] or "").split("#")[0].rstrip("/")
        if key and key not in seen:
            seen.add(key)
            out.append(r)
    return out


def search(query: str, n: int = 6) -> list:
    """Return up to n structured results [{title,url,snippet,source}] with DIRECT, fetchable URLs.
    Order favors engines whose links can actually be read: SearXNG (local, direct) → DuckDuckGo
    (direct) → Bing (resolvable click-redirect). Google News is excluded here (encoded links that
    don't resolve to full text) — use search_news() for freshness/discovery."""
    n = int(n)
    for engine in (_searxng, _ddg, _bing):
        try:
            hits = engine(query, n)
            if hits:
                return _dedup(hits)[:n]
        except Exception:
            continue
    return []


def search_news(query: str, n: int = 6) -> list:
    """Fresh headlines via Google News RSS — keyless, no rate limit. Links are encoded redirects
    (good for discovery/snippets; not reliably full-text readable), so keep this separate from the
    readable web search above."""
    try:
        return _dedup(_gnews(query, int(n)))[:int(n)]
    except Exception:
        return []


if __name__ == "__main__":
    import sys, json
    q = " ".join(sys.argv[1:]) or "what is retrieval augmented generation"
    res = search(q, 6)
    print(f"query: {q}\n{len(res)} results:")
    print(json.dumps(res, indent=2)[:2000])