tools/web_search_tool.py

# COST: ZERO — BM25 keyword scoring (rank-bm25, runs locally).
# Searches across ALL scraped AgentRax pages (home, pricing, about, help, blog, contact).
# Falls back to single-page snapshot if multi-page store is missing.
# A live HTTP fetch only occurs when cached content is stale (>60 min).
# No embedding model or LLM is invoked at any point in this tool.

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent.parent))

from dotenv import load_dotenv

load_dotenv()

from rank_bm25 import BM25Okapi

from scraper.content_store import (
    is_content_stale,
    is_multi_page_stale,
    load_all_pages,
    load_latest_content,
    save_all_pages,
    save_site_content,
)
from scraper.web_scraper import AGENTRAX_URL, scrape_all_pages, scrape_site

_TOP_K = 8       # results returned across all pages
_MIN_CHARS = 30  # minimum length to include a text fragment


def _page_label(url: str) -> str:
    """Return a short human-readable label for a page URL."""
    slug = url.rstrip("/").split("/")[-1]
    return slug if slug else "home"


def _build_corpus(pages: dict[str, dict]) -> tuple[list[str], list[str]]:
    """Return (corpus_texts, corpus_labels) from a {url: page_data} dict."""
    texts: list[str] = []
    labels: list[str] = []

    for url, content in pages.items():
        label = _page_label(url)
        headings = content.get("headings") or []
        body_text = content.get("body_text") or ""

        for h in headings:
            h = h.strip()
            if len(h) >= 5:
                texts.append(f"[{label}] {h}")
                labels.append(label)

        for para in body_text.split("\n"):
            para = para.strip()
            if len(para) >= _MIN_CHARS:
                texts.append(f"[{label}] {para}")
                labels.append(label)

    return texts, labels


async def search_agentrax_website(query: str) -> str:
    """Search all AgentRax website pages for information relevant to the user query.

    Searches across home, pricing, about, help, blog, and contact pages using BM25.
    Content is refreshed automatically every 60 minutes. Falls back to single-page
    snapshot if the multi-page store is unavailable.

    Args:
        query: The question or topic to search on the AgentRax website.

    Returns:
        Relevant extracted sections annotated with which page they came from.
    """
    # ── 1. Load / refresh multi-page content ─────────────────────────────────
    pages = load_all_pages()

    if is_multi_page_stale() or not pages:
        fresh = await scrape_all_pages()
        if fresh:
            save_all_pages(fresh)
            pages = {p["url"]: p for p in fresh}

    # ── 2. Fall back to single-page homepage snapshot ─────────────────────────
    if not pages:
        single = load_latest_content()
        if is_content_stale() or not single:
            single = await scrape_site(AGENTRAX_URL)
            if "error" not in single:
                save_site_content(single)
        if single and "error" not in single:
            pages = {single.get("url", AGENTRAX_URL): single}

    if not pages:
        return (
            "Error: could not retrieve AgentRax website content. "
            "Please try again in a moment."
        )

    # ── 3. Build cross-page searchable corpus ─────────────────────────────────
    corpus, labels = _build_corpus(pages)

    if not corpus:
        # Last resort: return description of first available page
        first = next(iter(pages.values()))
        title = first.get("title") or "AgentRax"
        desc = first.get("description") or ""
        return f"## {title}\n\n{desc}" if desc else f"## {title}\n\n(No content available.)"

    # ── 4. BM25 ranking ───────────────────────────────────────────────────────
    tokenized = [text.lower().split() for text in corpus]
    bm25 = BM25Okapi(tokenized)
    scores = bm25.get_scores(query.lower().split())

    top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:_TOP_K]
    # Preserve document order for readability
    top_indices_sorted = sorted(top_indices)

    # ── 5. Format output ──────────────────────────────────────────────────────
    lines = []
    for i in top_indices_sorted:
        lines.append(f"- {corpus[i]}")

    pages_scraped = ", ".join(sorted({_page_label(u) for u in pages}))
    header = f"## AgentRax — Pages searched: {pages_scraped}"

    return f"{header}\n\n### Most Relevant Content\n" + "\n".join(lines)