Add Docker setup for Hugging Face Spaces deployment

- Add Dockerfile with Python 3.11, uv, uid 1000, port 7860
- Add .dockerignore to exclude .env, logs, and runtime data
- Update README.md with app_port: 7860 for HF Spaces
- Add CORS origin for *.hf.space
- Add full project source: FastAPI RAG agent backend

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.dockerignore

@@ -0,0 +1,19 @@
+.env
+.git
+.gitignore
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.egg-info
+.venv
+venv
+node_modules
+*.log
+server.log
+server.err
+COST_NOTES.md
+documents/
+vector_store/
+cache/

.env.example

@@ -0,0 +1 @@
+OPENAI_API_KEY=your_key_here

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

COST_NOTES.md

@@ -0,0 +1,80 @@
+# Backend Cost Notes
+
+## Operations that cost OpenAI tokens
+
+| Operation | Model | When | Frequency |
+|---|---|---|---|
+| Final answer generation | gpt-4o-mini (chat) | Every **uncached** user query | Per unique question |
+| Document summarization | gpt-4o-mini (chat) | First call per document | Once per file; sidecar caches result |
+| Document ingestion (embedding) | text-embedding-3-small | When a document is indexed | Once per document; re-indexing re-bills |
+| Site re-indexing after scrape | text-embedding-3-small | When scheduler detects stale content | At most once per 60-minute window |
+
+## Operations that are free (zero OpenAI tokens)
+
+| Operation | How |
+|---|---|
+| Semantic cache lookup | Local sentence-transformers (`all-MiniLM-L6-v2`) + pure-Python cosine similarity |
+| Cache embedding on save | Same local model — no API call |
+| BM25 paragraph scoring | `rank-bm25` library, runs in-process |
+| Website scraping | Plain HTTP fetch + BeautifulSoup |
+| Content diff checking | `difflib.unified_diff` (stdlib) |
+| Staleness check | Timestamp arithmetic on cached JSON |
+| ChromaDB metadata queries | Reads stored metadata; no embedding triggered |
+| Document summarization (repeat) | Reads `.summary.json` sidecar; OpenAI never called |
+| Cache hit response | Returns stored answer string; agent never instantiated |
+
+## Token estimates per request
+
+### Uncached query (cache MISS)
+
+```
+System prompt:          ~100 tokens  (input)
+Tool schemas (3 tools): ~200 tokens  (input)
+Conversation history:   ~50 tokens   (input, typical 1-2 turns)
+User message:           ~20 tokens   (input)
+Tool call result        ~400 tokens  (input — BM25 top-5 paragraphs from website)
+  (search_agentrax_website)
+──────────────────────────────────────────────
+Total input:            ~770 tokens
+Agent response:         ~150 tokens  (output)
+──────────────────────────────────────────────
+Total per query:        ~920 tokens  → ~$0.00028 at gpt-4o-mini pricing
+```
+
+### Cached query (cache HIT)
+
+```
+OpenAI tokens:          0
+Local compute:          sentence-transformers inference + cosine scan over cache
+Latency:                <50 ms (no network call)
+Cost:                   $0.00
+```
+
+### Document summarization (first call)
+
+```
+System prompt:          ~50 tokens   (input)
+Document text:          up to 12,000 characters (~3,000 tokens, input)
+Summary output:         ~300 tokens  (output)
+──────────────────────────────────────────────
+Total first call:       ~3,350 tokens → ~$0.001 at gpt-4o-mini pricing
+Subsequent calls:       0 tokens (sidecar cache)
+```
+
+### Document ingestion (text-embedding-3-small)
+
+```
+Rate:                   $0.00002 per 1K tokens
+Typical document:       ~5,000 tokens across all chunks
+Cost per document:      ~$0.0001
+```
+
+## Cost reduction mechanisms in place
+
+1. **Semantic cache** — identical or near-identical questions (cosine ≥ 0.85) skip the agent entirely.
+2. **Scrape cache** — website content is served from disk for 60 minutes; HTTP fetch only on stale.
+3. **Summary sidecar** — document summaries are written to disk and never regenerated.
+4. **Staleness guard in scheduler** — `is_content_stale()` checked before any scrape or re-index.
+5. **Local embeddings for cache** — sentence-transformers runs on CPU, no API cost for cache operations.
+6. **ChromaDB singleton** — client opened once per process; no per-request reconnect overhead.
+7. **gpt-4o-mini** — ~15× cheaper than gpt-4o for equivalent tasks in this domain.

Dockerfile

@@ -0,0 +1,41 @@
+# Hugging Face Spaces requires port 7860 and uid 1000
+FROM python:3.11-slim
+
+# System dependencies for chromadb, lxml, sentence-transformers, unstructured
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    libxml2-dev \
+    libxslt-dev \
+    libsqlite3-dev \
+    libffi-dev \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv for fast dependency resolution
+RUN pip install --no-cache-dir uv
+
+# HF Spaces requires non-root user with uid 1000
+RUN useradd -m -u 1000 -s /bin/bash appuser
+
+WORKDIR /app
+
+# Install deps before copying source (better layer caching)
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev
+
+# Copy source code
+COPY --chown=appuser:appuser . .
+
+# Create writable runtime directories
+RUN mkdir -p vector_store cache documents \
+    && chown -R appuser:appuser /app
+
+USER appuser
+
+# Pre-download the sentence transformer model so first request is fast
+RUN uv run python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"
+
+EXPOSE 7860
+
+CMD ["uv", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,12 @@
----
-title: AgentraXhelpAgent
-emoji: 🦀
-colorFrom: yellow
-colorTo: indigo
-sdk: docker
-pinned: false
-short_description: usershelpAgent
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: AgentraXhelpAgent
+emoji: 🦀
+colorFrom: yellow
+colorTo: indigo
+sdk: docker
+app_port: 7860
+pinned: false
+short_description: usershelpAgent
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

agents/rag_agent.py

@@ -0,0 +1,105 @@
+import os
+import sys
+from pathlib import Path
+
+# The local backend/agents/ package shadows the openai-agents SDK `agents` package.
+# Fix: force .venv/Lib/site-packages to index 0, then clear the cached local entry.
+_venv_site = str((Path(__file__).parent.parent / ".venv" / "Lib" / "site-packages").resolve())
+if _venv_site in sys.path:
+    sys.path.remove(_venv_site)
+sys.path.insert(0, _venv_site)
+for _key in [k for k in sys.modules if k == "agents" or (k.startswith("agents.") and k != __name__)]:
+    del sys.modules[_key]
+
+from dotenv import load_dotenv
+load_dotenv(Path(__file__).parent.parent / ".env")
+
+from agents import Agent, RunConfig, function_tool  # openai-agents SDK
+from agents.models.openai_provider import OpenAIProvider
+
+from tools.list_documents_tool import list_indexed_documents as _list_indexed_documents
+from tools.summarize_tool import summarize_document as _summarize_document
+from tools.web_search_tool import search_agentrax_website as _search_agentrax_website
+
+_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
+_GEMINI_MODEL = "gemini-2.5-flash"
+
+# Module-level provider — created once, reused for every request
+_provider = OpenAIProvider(
+    api_key=os.environ.get("GEMINI_API_KEY", ""),
+    base_url=_GEMINI_BASE_URL,
+    use_responses=False,  # Gemini supports Chat Completions, not the Responses API
+)
+
+SYSTEM_PROMPT = """You are an expert assistant for AgentRax (https://agentrax.net/).
+
+RULES:
+- ALWAYS call search_agentrax_website first before answering any question about AgentRax
+- Never answer from general knowledge — only from tool results
+- Do NOT include source URLs or links in your response
+- If the website content does not contain the answer, say so clearly
+- Be concise — summarize tool output, do not dump raw text at the user
+"""
+
+
+@function_tool
+async def search_agentrax_website(query: str) -> str:
+    """Search the AgentRax website for information relevant to the user query.
+
+    PRIMARY tool — call this first for any question about AgentRax services,
+    pricing, features, or agents. Returns live or recently cached website content.
+
+    Args:
+        query: The question or topic to look up on the AgentRax website.
+
+    Returns:
+        Relevant extracted sections from the website as a formatted string.
+    """
+    return await _search_agentrax_website(query)
+
+
+@function_tool
+def summarize_document(file_path: str) -> str:
+    """Return a concise summary of a document, using a local cache when available.
+
+    Use when the user explicitly asks to summarise a specific document file.
+    Caches the result as a JSON sidecar — no repeated LLM cost on subsequent calls.
+
+    Args:
+        file_path: Path to the .pdf or .docx document to summarise.
+
+    Returns:
+        A plain-text summary, or an error message if the file is missing.
+    """
+    return _summarize_document(file_path)
+
+
+@function_tool
+def list_indexed_documents() -> list[str]:
+    """List all documents currently indexed in the vector store.
+
+    Use when the user asks what documents or files are available.
+    Queries ChromaDB metadata — no embeddings or LLM call involved.
+
+    Returns:
+        Sorted list of source file names, or an empty list if none are indexed.
+    """
+    return _list_indexed_documents()
+
+
+def create_rag_agent() -> Agent:
+    return Agent(
+        name="AgentRax Assistant",
+        instructions=SYSTEM_PROMPT,
+        model=_GEMINI_MODEL,
+        tools=[
+            search_agentrax_website,
+            summarize_document,
+            list_indexed_documents,
+        ],
+    )
+
+
+def get_run_config() -> RunConfig:
+    """Return a RunConfig that routes all model calls through Gemini."""
+    return RunConfig(model_provider=_provider)

api/routes.py

@@ -0,0 +1,220 @@
+import asyncio
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+# ── SDK shadowing fix ────────────────────────────────────────────────────────
+# Load rag_agent.py directly via importlib so its own SDK-fix code runs first,
+# setting sys.modules['agents'] to the openai-agents SDK.  After exec_module()
+# returns, `from agents import Runner` resolves to the SDK, not the local package.
+_rag_agent_path = Path(__file__).parent.parent / "agents" / "rag_agent.py"
+_spec = importlib.util.spec_from_file_location("agents.rag_agent", str(_rag_agent_path))
+_rag_mod = importlib.util.module_from_spec(_spec)
+sys.modules["agents.rag_agent"] = _rag_mod
+_spec.loader.exec_module(_rag_mod)  # runs rag_agent.py → SDK is now sys.modules['agents']
+create_rag_agent = _rag_mod.create_rag_agent
+get_run_config = _rag_mod.get_run_config
+
+# Now safe to import SDK symbols — agents points to the SDK in sys.modules
+from agents import Runner  # noqa: E402
+from agents.stream_events import RawResponsesStreamEvent  # noqa: E402
+from fastapi import APIRouter, HTTPException  # noqa: E402
+from fastapi.responses import JSONResponse, StreamingResponse  # noqa: E402
+from openai.types.chat import ChatCompletionChunk  # noqa: E402
+from openai.types.responses import ResponseTextDeltaEvent  # noqa: E402
+from pydantic import BaseModel  # noqa: E402
+
+from cache.query_cache import _cache as _disk_cache  # noqa: E402
+from cache.query_cache import find_similar_cached, get_cache_stats, save_to_cache  # noqa: E402
+from db.history import list_sessions, load_history, save_turn  # noqa: E402
+from ingestion import ingest_all_documents  # noqa: E402
+from scraper.content_store import is_content_stale, load_latest_content  # noqa: E402
+from tools.list_documents_tool import list_indexed_documents  # noqa: E402
+
+router = APIRouter()
+
+
+# ── request / response models ────────────────────────────────────────────────
+
+class HistoryMessage(BaseModel):
+    role: str        # "user" | "assistant"
+    content: str
+
+
+class ChatRequest(BaseModel):
+    message: str
+    session_id: str = ""
+    history: list[HistoryMessage] = []
+
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+def _build_input(db_history: list[dict], message: str) -> list[dict]:
+    """Merge DB history with the current message into Agents SDK input format."""
+    messages = [{"role": t["role"], "content": t["content"]} for t in db_history]
+    messages.append({"role": "user", "content": message})
+    return messages
+
+
+async def _stream_and_cache(request: ChatRequest, db_history: list[dict]):
+    """Stream token deltas as SSE, then persist the turn and save to cache."""
+    full_text: list[str] = []
+    try:
+        agent = create_rag_agent()
+        result = Runner.run_streamed(
+            agent,
+            _build_input(db_history, request.message),
+            run_config=get_run_config(),
+        )
+
+        async for event in result.stream_events():
+            if not isinstance(event, RawResponsesStreamEvent):
+                continue
+            data = event.data
+            delta: str | None = None
+            if isinstance(data, ResponseTextDeltaEvent):
+                # Responses API (OpenAI native)
+                delta = data.delta
+            elif isinstance(data, ChatCompletionChunk):
+                # Chat Completions API (Gemini compatible)
+                delta = data.choices[0].delta.content if data.choices else None
+            if delta:
+                full_text.append(delta)
+                yield f"data: {json.dumps({'delta': delta})}\n\n"
+
+    except Exception as exc:
+        yield f"data: {json.dumps({'error': str(exc)})}\n\n"
+
+    finally:
+        yield "data: [DONE]\n\n"
+        if full_text:
+            response_text = "".join(full_text)
+            save_to_cache(request.message, response_text, [])
+            if request.session_id:
+                await asyncio.to_thread(save_turn, request.session_id, request.message, response_text)
+
+
+# ── routes ───────────────────────────────────────────────────────────────────
+
+@router.post("/chat")
+async def chat(request: ChatRequest):
+    """Return AgentRax answer, served from semantic cache when possible.
+
+    Flow:
+    1. Load conversation history from PostgreSQL by session_id.
+    2. Check semantic cache (threshold 0.92). On HIT: return immediately, save turn.
+    3. On MISS: stream agent response, save to cache + DB on completion.
+
+    Cache HIT  → JSON  { answer, sources }  with X-Cache: HIT  (zero LLM cost)
+    Cache MISS → SSE token stream            with X-Cache: MISS
+    """
+    # Load DB history first — used by both HIT and MISS paths for context
+    db_history = await asyncio.to_thread(load_history, request.session_id)
+
+    cached = find_similar_cached(request.message)
+    if cached:
+        # Still persist the cached answer so history is complete
+        if request.session_id:
+            await asyncio.to_thread(
+                save_turn, request.session_id, request.message, cached["answer"]
+            )
+        return JSONResponse(
+            content={"answer": cached["answer"], "sources": cached.get("sources", [])},
+            headers={"X-Cache": "HIT"},
+        )
+
+    return StreamingResponse(
+        _stream_and_cache(request, db_history),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+            "X-Cache": "MISS",
+        },
+    )
+
+
+@router.get("/sessions")
+async def get_sessions():
+    """Return all sessions ordered by last activity, with title and message count."""
+    try:
+        sessions = await asyncio.to_thread(list_sessions)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+    # Serialize datetime objects to ISO strings
+    return {
+        "sessions": [
+            {**s, "last_active": s["last_active"].isoformat() if s.get("last_active") else None}
+            for s in sessions
+        ]
+    }
+
+
+@router.get("/history/{session_id}")
+async def get_history(session_id: str):
+    """Return full conversation history for a session from PostgreSQL."""
+    try:
+        turns = await asyncio.to_thread(load_history, session_id, 100)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+    return {"session_id": session_id, "history": turns}
+
+
+@router.get("/cache/stats")
+async def cache_stats():
+    """Return hit counts and top queries from the semantic cache."""
+    return get_cache_stats()
+
+
+@router.post("/cache/clear")
+async def cache_clear():
+    """Wipe all entries from the semantic cache (admin use only)."""
+    _disk_cache.clear()
+    return {"status": "ok", "message": "Cache cleared."}
+
+
+@router.get("/site/status")
+async def site_status():
+    """Return metadata about the last scraped AgentRax snapshot."""
+    content = load_latest_content()
+    if not content:
+        return {"status": "no_data", "is_stale": True}
+    return {
+        "title": content.get("title"),
+        "url": content.get("url"),
+        "scraped_at": content.get("scraped_at") or content.get("saved_at"),
+        "is_stale": is_content_stale(),
+    }
+
+
+@router.post("/ingest")
+async def ingest():
+    """Scan documents/ and index all .pdf and .docx files into ChromaDB."""
+    try:
+        result = await asyncio.to_thread(ingest_all_documents)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+    return {
+        "status": "ok",
+        "documents_indexed": result.get("ingested", 0),
+        "total_chunks": result.get("total_chunks", 0),
+    }
+
+
+@router.get("/documents")
+async def documents():
+    """Return all source file names currently indexed in the vector store."""
+    try:
+        docs = await asyncio.to_thread(list_indexed_documents)
+    except Exception as exc:
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+    return {"documents": docs}
+
+
+@router.get("/health")
+async def health():
+    """Liveness check."""
+    return {"status": "ok"}

cache/query_cache.py

@@ -0,0 +1,90 @@
+# COST: ZERO OpenAI tokens.
+# All embedding uses sentence-transformers 'all-MiniLM-L6-v2' running locally.
+# Similarity scoring is pure-Python cosine distance against diskcache entries.
+# No network call is made during cache lookup or save.
+
+from datetime import datetime, timezone
+from pathlib import Path
+import math
+import uuid
+
+import diskcache
+
+STORE_PATH = Path(__file__).parent / "query_store"
+_cache = diskcache.Cache(str(STORE_PATH))
+
+# --- model singleton ---
+_model = None
+
+
+def _get_model():
+    global _model
+    if _model is None:
+        from sentence_transformers import SentenceTransformer
+        _model = SentenceTransformer("all-MiniLM-L6-v2")
+    return _model
+
+
+# --- math helpers ---
+
+def _cosine_similarity(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b))
+    mag_a = math.sqrt(sum(x * x for x in a))
+    mag_b = math.sqrt(sum(y * y for y in b))
+    if mag_a == 0 or mag_b == 0:
+        return 0.0
+    return dot / (mag_a * mag_b)
+
+
+# --- public API ---
+
+def embed_query_local(query: str) -> list[float]:
+    return _get_model().encode(query, convert_to_numpy=True).tolist()
+
+
+def find_similar_cached(query: str, threshold: float = 0.92) -> dict | None:
+    query_emb = embed_query_local(query)
+
+    best_key = None
+    best_score = -1.0
+
+    for key in _cache:
+        entry = _cache[key]
+        score = _cosine_similarity(query_emb, entry["query_embedding"])
+        if score > best_score:
+            best_score = score
+            best_key = key
+
+    if best_key is None or best_score < threshold:
+        return None
+
+    entry = _cache[best_key]
+    entry["hit_count"] += 1
+    _cache[best_key] = entry
+    return entry
+
+
+def save_to_cache(query: str, answer: str, sources: list[str]) -> None:
+    entry = {
+        "query": query,
+        "query_embedding": embed_query_local(query),
+        "answer": answer,
+        "sources": sources,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "hit_count": 0,
+    }
+    _cache[str(uuid.uuid4())] = entry
+
+
+def get_cache_stats() -> dict:
+    entries = [_cache[k] for k in _cache]
+    total_hits = sum(e["hit_count"] for e in entries)
+    most_asked = [
+        e["query"]
+        for e in sorted(entries, key=lambda e: e["hit_count"], reverse=True)[:5]
+    ]
+    return {
+        "total_entries": len(entries),
+        "total_hits": total_hits,
+        "most_asked": most_asked,
+    }

chroma.py

@@ -0,0 +1,34 @@
+"""Singleton ChromaDB client and collection.
+
+All modules that need the vector store import get_collection() from here.
+The PersistentClient and collection are created once on first call and
+reused for the lifetime of the process.
+
+COST: ZERO external API tokens.
+Embeddings use sentence-transformers 'all-MiniLM-L6-v2' running locally —
+the same model used by the semantic query cache.
+"""
+from pathlib import Path
+from typing import Optional
+
+import chromadb
+from chromadb.utils.embedding_functions import SentenceTransformerEmbeddingFunction
+
+VECTOR_STORE_DIR = Path(__file__).parent / "vector_store"
+COLLECTION_NAME = "rag_documents"
+
+_client: Optional[chromadb.Client] = None
+_collection: Optional[chromadb.Collection] = None
+
+
+def get_collection() -> chromadb.Collection:
+    """Return the shared ChromaDB collection, creating it on first call."""
+    global _client, _collection
+    if _collection is None:
+        _client = chromadb.PersistentClient(path=str(VECTOR_STORE_DIR))
+        ef = SentenceTransformerEmbeddingFunction(model_name="all-MiniLM-L6-v2")
+        _collection = _client.get_or_create_collection(
+            name=COLLECTION_NAME,
+            embedding_function=ef,
+        )
+    return _collection

db/history.py

@@ -0,0 +1,130 @@
+"""
+PostgreSQL-backed conversation history.
+
+Each session_id maps to an ordered list of {role, content} turns.
+The table is created on first call to init_db() — safe to run on every startup.
+"""
+import logging
+import os
+from pathlib import Path
+
+import psycopg2
+import psycopg2.extras
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).parent.parent / ".env")
+
+logger = logging.getLogger(__name__)
+
+_DATABASE_URL = os.environ.get(
+    "DATABASE_URL",
+    "postgresql://neondb_owner:npg_8QHOFkvXuxj6@ep-flat-sea-and50v08-pooler.c-6.us-east-1.aws.neon.tech/neondb?sslmode=require&channel_binding=require",
+)
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS chat_history (
+    id          BIGSERIAL PRIMARY KEY,
+    session_id  TEXT        NOT NULL,
+    role        TEXT        NOT NULL CHECK (role IN ('user', 'assistant')),
+    content     TEXT        NOT NULL,
+    created_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+CREATE INDEX IF NOT EXISTS idx_chat_history_session
+    ON chat_history (session_id, created_at);
+"""
+
+
+def _connect() -> psycopg2.extensions.connection:
+    return psycopg2.connect(_DATABASE_URL)
+
+
+def init_db() -> None:
+    """Create table and index if they don't exist. Called once at app startup."""
+    try:
+        with _connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(_CREATE_TABLE)
+        logger.info("chat_history table ready.")
+    except Exception:
+        logger.exception("Failed to initialise chat_history table.")
+        raise
+
+
+def save_turn(session_id: str, user_message: str, assistant_message: str) -> None:
+    """Persist a single Q-A turn for the given session."""
+    if not session_id:
+        return
+    try:
+        with _connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    "INSERT INTO chat_history (session_id, role, content) VALUES (%s, %s, %s), (%s, %s, %s)",
+                    (session_id, "user", user_message,
+                     session_id, "assistant", assistant_message),
+                )
+    except Exception:
+        logger.exception("Failed to save conversation turn for session %s.", session_id)
+
+
+def list_sessions(limit: int = 50) -> list[dict]:
+    """Return the most recent sessions with their title (first user message) and metadata."""
+    try:
+        with _connect() as conn:
+            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+                cur.execute(
+                    """
+                    WITH first_user AS (
+                        SELECT DISTINCT ON (session_id)
+                            session_id, content AS title
+                        FROM chat_history
+                        WHERE role = 'user'
+                        ORDER BY session_id, created_at ASC
+                    ),
+                    agg AS (
+                        SELECT session_id,
+                               MAX(created_at)  AS last_active,
+                               COUNT(*)         AS message_count
+                        FROM chat_history
+                        GROUP BY session_id
+                    )
+                    SELECT f.session_id, f.title, a.last_active, a.message_count
+                    FROM first_user f
+                    JOIN agg a ON f.session_id = a.session_id
+                    ORDER BY a.last_active DESC
+                    LIMIT %s
+                    """,
+                    (limit,),
+                )
+                return [dict(row) for row in cur.fetchall()]
+    except Exception:
+        logger.exception("Failed to list sessions.")
+        return []
+
+
+def load_history(session_id: str, limit: int = 20) -> list[dict]:
+    """Return the most recent `limit` turns for the session, oldest first.
+
+    Returns a list of {role, content} dicts compatible with the Agents SDK input format.
+    """
+    if not session_id:
+        return []
+    try:
+        with _connect() as conn:
+            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
+                cur.execute(
+                    """
+                    SELECT role, content FROM (
+                        SELECT role, content, created_at
+                        FROM chat_history
+                        WHERE session_id = %s
+                        ORDER BY created_at DESC
+                        LIMIT %s
+                    ) sub
+                    ORDER BY created_at ASC
+                    """,
+                    (session_id, limit),
+                )
+                return [dict(row) for row in cur.fetchall()]
+    except Exception:
+        logger.exception("Failed to load history for session %s.", session_id)
+        return []

ingestion.py

@@ -0,0 +1,157 @@
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from chroma import get_collection  # noqa: E402 — after load_dotenv
+
+DOCUMENTS_DIR = Path(__file__).parent / "documents"
+
+
+def parse_document(file_path: str | Path) -> list[dict[str, Any]]:
+    """Return list of {text, page_number} dicts from a .pdf or .docx file."""
+    file_path = Path(file_path)
+    suffix = file_path.suffix.lower()
+
+    if suffix == ".pdf":
+        from pypdf import PdfReader
+
+        reader = PdfReader(str(file_path))
+        pages = []
+        for i, page in enumerate(reader.pages):
+            text = page.extract_text() or ""
+            if text.strip():
+                pages.append({"text": text, "page_number": i + 1})
+        return pages
+
+    if suffix == ".docx":
+        from docx import Document
+
+        doc = Document(str(file_path))
+        full_text = "\n".join(p.text for p in doc.paragraphs if p.text.strip())
+        return [{"text": full_text, "page_number": None}]
+
+    raise ValueError(f"Unsupported file type: {suffix!r}. Only .pdf and .docx are supported.")
+
+
+def chunk_text(
+    text: str,
+    chunk_size: int = 500,
+    overlap: int = 50,
+) -> list[str]:
+    """Split text into overlapping word-based chunks."""
+    words = text.split()
+    if not words:
+        return []
+
+    chunks: list[str] = []
+    start = 0
+    while start < len(words):
+        end = min(start + chunk_size, len(words))
+        chunks.append(" ".join(words[start:end]))
+        if end == len(words):
+            break
+        start += chunk_size - overlap
+
+    return chunks
+
+
+def embed_and_store(
+    chunks: list[str],
+    metadata: dict[str, Any],
+) -> None:
+    """Embed chunks and upsert them into ChromaDB with per-chunk metadata."""
+    if not chunks:
+        return
+
+    collection = get_collection()
+
+    source_file = metadata.get("source_file", "unknown")
+    page_number = metadata.get("page_number")  # may be None for docx
+
+    ids = [
+        f"{source_file}__p{page_number}__c{i}" if page_number is not None else f"{source_file}__c{i}"
+        for i in range(len(chunks))
+    ]
+
+    chunk_metadata = [
+        {
+            "source_file": source_file,
+            "chunk_index": i,
+            **({"page_number": page_number} if page_number is not None else {}),
+        }
+        for i in range(len(chunks))
+    ]
+
+    collection.upsert(documents=chunks, metadatas=chunk_metadata, ids=ids)
+
+
+def ingest_all_documents() -> dict[str, Any]:
+    """Scan documents/ folder and run the full ingestion pipeline."""
+    supported = {".pdf", ".docx"}
+    files = [f for f in DOCUMENTS_DIR.iterdir() if f.suffix.lower() in supported]
+
+    if not files:
+        print("No documents found in", DOCUMENTS_DIR)
+        return {"ingested": 0, "files": []}
+
+    results: list[str] = []
+    total_chunks = 0
+
+    for file_path in files:
+        print(f"Ingesting: {file_path.name}")
+        try:
+            pages = parse_document(file_path)
+            file_chunks = 0
+
+            for page in pages:
+                chunks = chunk_text(page["text"])
+                embed_and_store(
+                    chunks,
+                    metadata={
+                        "source_file": file_path.name,
+                        "page_number": page["page_number"],
+                    },
+                )
+                file_chunks += len(chunks)
+
+            total_chunks += file_chunks
+            results.append(file_path.name)
+            print(f"  -> {file_chunks} chunks stored")
+
+        except Exception as e:
+            print(f"  -> ERROR: {e}")
+
+    print(f"\nDone. {len(results)} file(s), {total_chunks} total chunks.")
+    return {"ingested": len(results), "total_chunks": total_chunks, "files": results}
+
+
+def ingest_scraped_content(data: dict) -> None:
+    """Chunk and upsert scraped website content into ChromaDB."""
+    parts = []
+    if data.get("title"):
+        parts.append(data["title"])
+    if data.get("description"):
+        parts.append(data["description"])
+    if data.get("headings"):
+        parts.append("\n".join(data["headings"]))
+    if data.get("body_text"):
+        parts.append(data["body_text"])
+
+    full_text = "\n\n".join(parts)
+    chunks = chunk_text(full_text)
+    if not chunks:
+        return
+
+    url = data.get("url", "scraped")
+    scraped_at = data.get("scraped_at", "")
+    embed_and_store(
+        chunks,
+        metadata={"source_file": url, "scraped_at": scraped_at},
+    )
+
+
+if __name__ == "__main__":
+    ingest_all_documents()

main.py

@@ -0,0 +1,39 @@
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).parent / ".env")
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from api.routes import router
+from db.history import init_db
+from scraper.scheduler import start_scheduler, stop_scheduler
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    init_db()
+    start_scheduler()
+    yield
+    stop_scheduler()
+
+
+app = FastAPI(title="RAG Agent API", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3000", "https://*.hf.space"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(router)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "rag-agent-backend"
+version = "0.1.0"
+description = "RAG Agent backend"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi",
+    "uvicorn[standard]",
+    "openai",
+    "openai-agents",
+    "chromadb",
+    "pypdf",
+    "python-docx",
+    "unstructured",
+    "python-dotenv",
+    "tiktoken",
+    "httpx",
+    "beautifulsoup4",
+    "lxml",
+    "apscheduler",
+    "diskcache",
+    "sentence-transformers",
+    "rank-bm25",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[dependency-groups]
+dev = []

scraper/_test_store.json

@@ -0,0 +1,42 @@
+[
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-28T20:56:16.212258+00:00",
+    "saved_at": "2026-04-28T20:56:16.212788+00:00"
+  }
+]

scraper/content_store.py

@@ -0,0 +1,71 @@
+import difflib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+STORE_PATH = Path(__file__).parent / "site_content.json"
+MAX_SNAPSHOTS = 10
+
+
+def _load_store() -> list[dict]:
+    if not STORE_PATH.exists():
+        return []
+    with STORE_PATH.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _write_store(snapshots: list[dict]) -> None:
+    with STORE_PATH.open("w", encoding="utf-8") as f:
+        json.dump(snapshots, f, ensure_ascii=False, indent=2)
+
+
+def save_site_content(data: dict) -> None:
+    snapshots = _load_store()
+    entry = {**data, "saved_at": datetime.now(timezone.utc).isoformat()}
+    snapshots.append(entry)
+    snapshots = snapshots[-MAX_SNAPSHOTS:]
+    _write_store(snapshots)
+
+
+def load_latest_content() -> dict:
+    snapshots = _load_store()
+    if not snapshots:
+        return {}
+    return snapshots[-1]
+
+
+def get_content_diff(snapshot_1: dict, snapshot_2: dict) -> dict:
+    text_a = (snapshot_1.get("body_text") or "").splitlines()
+    text_b = (snapshot_2.get("body_text") or "").splitlines()
+
+    diff_lines = list(
+        difflib.unified_diff(text_a, text_b, lineterm="", n=2)
+    )
+
+    if not diff_lines:
+        return {"is_changed": False, "summary_of_changes": "No changes detected."}
+
+    added = [l[1:] for l in diff_lines if l.startswith("+") and not l.startswith("+++")]
+    removed = [l[1:] for l in diff_lines if l.startswith("-") and not l.startswith("---")]
+
+    parts = []
+    if removed:
+        parts.append(f"Removed ({len(removed)} line(s)):\n" + "\n".join(f"  - {l}" for l in removed[:5]))
+    if added:
+        parts.append(f"Added ({len(added)} line(s)):\n" + "\n".join(f"  + {l}" for l in added[:5]))
+    if len(removed) > 5 or len(added) > 5:
+        parts.append("(truncated — showing first 5 lines per section)")
+
+    return {"is_changed": True, "summary_of_changes": "\n".join(parts)}
+
+
+def is_content_stale(max_age_minutes: int = 60) -> bool:
+    latest = load_latest_content()
+    if not latest:
+        return True
+    saved_at_str = latest.get("saved_at") or latest.get("scraped_at")
+    if not saved_at_str:
+        return True
+    saved_at = datetime.fromisoformat(saved_at_str)
+    age_minutes = (datetime.now(timezone.utc) - saved_at).total_seconds() / 60
+    return age_minutes > max_age_minutes

scraper/scheduler.py

@@ -0,0 +1,70 @@
+# COST: ZERO OpenAI chat tokens.
+# Scraping is plain HTTP + BeautifulSoup. Diff checking uses difflib (stdlib).
+# is_content_stale() is checked first — if content is fresh the job returns
+# immediately, skipping both the HTTP fetch and re-indexing.
+# Re-indexing (ingest_scraped_content) calls text-embedding-3-small for new
+# chunks only — one-time embedding cost, no chat LLM.
+
+import logging
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from ingestion import ingest_scraped_content
+from scraper.content_store import (
+    get_content_diff,
+    is_content_stale,
+    load_latest_content,
+    save_site_content,
+)
+from scraper.web_scraper import AGENTRAX_URL, scrape_site
+
+logger = logging.getLogger(__name__)
+
+_scheduler: AsyncIOScheduler | None = None
+
+
+async def refresh_agentrax_content() -> None:
+    if not is_content_stale():
+        logger.info("Site content is fresh — skipping refresh.")
+        return
+
+    logger.info("Scraping %s ...", AGENTRAX_URL)
+    data = await scrape_site(AGENTRAX_URL)
+
+    if "error" in data:
+        logger.error("Scrape failed: %s", data["error"])
+        return
+
+    previous = load_latest_content()
+    if previous:
+        diff = get_content_diff(previous, data)
+        if diff["is_changed"]:
+            logger.warning("SITE UPDATED: %s", diff["summary_of_changes"])
+
+    save_site_content(data)
+    ingest_scraped_content(data)
+    logger.info("Site content refreshed and re-indexed.")
+
+
+def start_scheduler() -> None:
+    global _scheduler
+    _scheduler = AsyncIOScheduler()
+    _scheduler.add_job(
+        refresh_agentrax_content,
+        trigger="interval",
+        minutes=60,
+        id="refresh_agentrax",
+        replace_existing=True,
+    )
+    _scheduler.start()
+    logger.info("Scheduler started — refresh every 60 minutes.")
+
+    import asyncio
+    asyncio.get_event_loop().create_task(refresh_agentrax_content())
+
+
+def stop_scheduler() -> None:
+    global _scheduler
+    if _scheduler and _scheduler.running:
+        _scheduler.shutdown(wait=False)
+        logger.info("Scheduler stopped.")

scraper/site_content.json

@@ -0,0 +1,202 @@
+[
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-28T20:49:55.863991+00:00",
+    "saved_at": "2026-04-28T20:49:55.863991+00:00"
+  },
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment\nNew line added to site.",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-28T20:49:55.863991+00:00",
+    "saved_at": "2026-04-28T20:49:55.886537+00:00"
+  },
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-30T00:49:26.376898+00:00",
+    "saved_at": "2026-04-30T00:49:26.391892+00:00"
+  },
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-30T01:49:46.518110+00:00",
+    "saved_at": "2026-04-30T01:49:46.518110+00:00"
+  },
+  {
+    "title": "Build Autonomous AI Agents in 30 Seconds | No-Code AI Platform",
+    "description": "Create no-code AI agents in 30 seconds with Agentrax. Build autonomous, adaptive AI agents for your business without coding. From 3 hours of manual coding to instant automation for SMEs.",
+    "headings": [
+      "Build Autonomous AI Agents Fasterwith Agentrax",
+      "See It in Action",
+      "Harness the Power of No-Code AI, Making Agent Building Intuitive and Effective for All Skill Levels",
+      "SEO goal setting",
+      "User-friendly dashboard",
+      "Visual reports",
+      "Smart Keyword Generator",
+      "Features",
+      "Intuitive No-Code Builder",
+      "Autonomous Agent Testing",
+      "Deployment Wizard",
+      "Visual Agent Analytics",
+      "AI Goal Setting",
+      "One-Click Agent Creation",
+      "Smart Agent Generator",
+      "Automated Agent Alerts",
+      "Competitor AI Insights",
+      "Pricing",
+      "Pro Daily",
+      "Pro",
+      "Enterprise",
+      "Start Building Your First Agent Today.",
+      "Latest from Our Blog",
+      "Building AI Agents Without Coding",
+      "Optimizing Agent Performance",
+      "Real-World Agent Applications",
+      "Product Updates",
+      "Advanced Agent Analytics Dashboard",
+      "Multi-Model Support",
+      "Enhanced Deployment Options"
+    ],
+    "body_text": "Create no-code AI agents in 30 seconds – autonomous, adaptive, and ready for action. From 3 hours of manual coding to instant automation for SMEs\nWatch how Agentrax transforms your workflow with autonomous AI agents\nHelps you set and achieve SEO goals with guided assistance.\nPerform complex SEO audits and optimizations with a single click.\nVisual insights into your site's performance.\nAutomatic suggestions and the best keywords to target.\nDrag-drop for autonomous agents.\nSelf-optimizes workflows in real-time.\nEasy API, sharable links, self-hosted setup\nTrack performance and adaptations.\nSet tasks for agents with guided assistance.\n30-sec builds with OpenAI SDK.\nAutomatic suggestions for tasks/triggers.\nNotifications on performance/health.\nCompare agents with market benchmarks.\nChoose the Right Plan to Meet Your AI Agent Needs and Start Building Today.\nNo credit card required • 7-days free trial\nStay updated with the latest news, tutorials, and insights about AI agents and automation.\nDiscover how our no-code platform enables anyone to create powerful AI agents in minutes.\nLearn best practices for maximizing the efficiency and effectiveness of your AI agents.\nExplore how businesses are leveraging AI agents to automate workflows and increase productivity.\nSee what's new in Agentrax and how we're continuously improving our platform.\nIntroducing comprehensive analytics to track your agent's performance, usage, and effectiveness with detailed metrics and visualizations.\nNow supports OpenAI, Google Gemini, and xAI Grok models with seamless switching between providers based on your needs.\nNew deployment options including direct cloud hosting, Docker containers, and API endpoints for seamless integration.\nNo-Code Agent Builder\nAutomated Deployments\nBasic Autonomous Agents\nAPI Endpoint Access\nEmail Support\nNo-Code Agent Builder\nAutomated Meta Tags\nAutonomous Agent Testing\nDeployment Wizard\nVisual Agent Analytics\nAI Goal Setting\nPriority Support\nAdvanced Integrations\nEverything in Pro\nUnlimited Agent Creation\nCustom AI Models\nDedicated Account Manager\n24/7 Priority Support\nMulti-User Access\nAdvanced API Access\nCustom Integrations\nSLA Guarantee\nOn-premise Deployment",
+    "url": "https://agentrax.net/",
+    "scraped_at": "2026-04-30T03:19:45.326252+00:00",
+    "saved_at": "2026-04-30T03:19:45.328781+00:00"
+  }
+]

scraper/web_scraper.py

@@ -0,0 +1,74 @@
+from datetime import datetime, timezone
+from bs4 import BeautifulSoup
+import httpx
+
+AGENTRAX_URL = "https://agentrax.net/"
+
+_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    )
+}
+
+_NOISE_TAGS = {"nav", "footer", "header", "script", "style", "aside"}
+_NOISE_CLASSES = {
+    "ad", "ads", "advert", "advertisement", "banner", "cookie",
+    "popup", "modal", "sidebar", "widget", "promo",
+}
+
+
+async def fetch_page(url: str) -> str:
+    async with httpx.AsyncClient(headers=_HEADERS, timeout=15, follow_redirects=True) as client:
+        response = await client.get(url)
+        response.raise_for_status()
+        return response.text
+
+
+def extract_clean_text(html: str, url: str) -> dict:
+    soup = BeautifulSoup(html, "lxml")
+
+    for tag in soup.find_all(_NOISE_TAGS):
+        tag.decompose()
+
+    for tag in soup.find_all(True):
+        classes = set(tag.get("class") or [])
+        if classes & _NOISE_CLASSES:
+            tag.decompose()
+
+    title = soup.title.get_text(strip=True) if soup.title else ""
+
+    description = ""
+    meta = soup.find("meta", attrs={"name": "description"})
+    if meta:
+        description = meta.get("content", "").strip()
+
+    headings = [
+        tag.get_text(strip=True)
+        for tag in soup.find_all(["h1", "h2", "h3"])
+        if tag.get_text(strip=True)
+    ]
+
+    paragraphs = [p.get_text(strip=True) for p in soup.find_all("p") if p.get_text(strip=True)]
+    list_items = [li.get_text(strip=True) for li in soup.find_all("li") if li.get_text(strip=True)]
+
+    body_parts = paragraphs + list_items
+    body_text = "\n".join(body_parts)
+
+    return {
+        "title": title,
+        "description": description,
+        "headings": headings,
+        "body_text": body_text,
+        "url": url,
+        "scraped_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+async def scrape_site(url: str = AGENTRAX_URL) -> dict:
+    try:
+        html = await fetch_page(url)
+        return extract_clean_text(html, url)
+    except Exception as exc:
+        return {"error": str(exc), "url": url}

tools/document_loader_tool.py

@@ -0,0 +1,64 @@
+# COST: OpenAI text-embedding-3-small called during indexing (one-time per document).
+# No chat LLM is invoked. Embedding cost is ~$0.00002/1K tokens — negligible for
+# typical documents. Upsert semantics mean re-indexing the same file is idempotent
+# but does re-embed (and re-bill) all chunks.
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from ingestion import chunk_text, embed_and_store, parse_document
+
+
+def load_and_index_document(file_path: str) -> str:
+    """Parse, chunk, embed, and store a single document into the vector store.
+
+    Accepts a .pdf or .docx file, extracts text (preserving per-page structure
+    for PDFs), splits it into overlapping 500-word chunks with a 50-word overlap,
+    and upserts every chunk with source metadata into the persistent ChromaDB
+    collection.  Calling this function on an already-indexed file is safe — all
+    chunks are upserted so duplicates are automatically overwritten.
+
+    Args:
+        file_path: Absolute or relative path to the document to ingest.
+                   Must be a .pdf or .docx file.
+
+    Returns:
+        A human-readable confirmation string such as:
+            "Successfully indexed 'report.pdf': 42 chunks stored across 10 page(s)."
+        If the file cannot be parsed an error message is returned instead of
+        raising, so the agent can surface it gracefully.
+    """
+    path = Path(file_path)
+
+    if not path.exists():
+        return f"Error: file not found at '{file_path}'."
+
+    try:
+        pages = parse_document(path)
+    except ValueError as exc:
+        return f"Error: {exc}"
+
+    total_chunks = 0
+    for page in pages:
+        chunks = chunk_text(page["text"])
+        embed_and_store(
+            chunks,
+            metadata={
+                "source_file": path.name,
+                "page_number": page["page_number"],
+            },
+        )
+        total_chunks += len(chunks)
+
+    page_count = len(pages)
+    page_label = f"{page_count} page(s)" if pages[0]["page_number"] is not None else "1 section"
+    return (
+        f"Successfully indexed '{path.name}': "
+        f"{total_chunks} chunks stored across {page_label}."
+    )

tools/list_documents_tool.py

@@ -0,0 +1,42 @@
+# COST: ZERO OpenAI tokens.
+# Reads ChromaDB metadata only — no embedding or similarity search performed.
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from chroma import get_collection  # noqa: E402
+
+
+def list_indexed_documents() -> list[str]:
+    """Return the names of all documents that have been indexed in the vector store.
+
+    Queries the ChromaDB collection metadata to collect every unique
+    'source_file' value without loading any embeddings or performing any
+    similarity search.  No LLM call is made.
+
+    This is useful for letting the agent know which documents are available
+    before deciding whether to retrieve chunks or ask the user to upload more
+    files.
+
+    Returns:
+        A sorted list of unique source file names (e.g. ['manual.pdf', 'notes.docx']).
+        Returns an empty list if no documents have been indexed yet.
+    """
+    collection = get_collection()
+    if collection.count() == 0:
+        return []
+
+    result = collection.get(include=["metadatas"])
+    seen: set[str] = set()
+    for meta in result["metadatas"]:
+        source = meta.get("source_file")
+        if source:
+            seen.add(source)
+
+    return sorted(seen)

tools/retrieval_tool.py

@@ -0,0 +1,52 @@
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from chroma import get_collection  # noqa: E402
+
+
+def retrieve_chunks(query: str, top_k: int = 5) -> list[str]:
+    """Search the vector store for document chunks most relevant to the query.
+
+    Embeds the query using the same OpenAI embedding model used during ingestion,
+    performs a cosine-similarity search against the ChromaDB collection, and
+    returns the top-k matching text chunks formatted with source metadata so
+    the caller can cite the origin of each passage.
+
+    Args:
+        query:  The natural-language question or search string to look up.
+        top_k:  Maximum number of chunks to return (default 5).
+
+    Returns:
+        A list of strings, each formatted as:
+            "[source: <filename>, page: <n>, chunk: <i>]\\n<chunk text>"
+        Page number is omitted for DOCX files where it is unavailable.
+    """
+    collection = get_collection()
+    count = collection.count()
+    if count == 0:
+        return ["No documents have been indexed yet. Please ingest documents first."]
+
+    results = collection.query(
+        query_texts=[query],
+        n_results=min(top_k, count),
+        include=["documents", "metadatas"],
+    )
+
+    chunks: list[str] = []
+    for doc, meta in zip(results["documents"][0], results["metadatas"][0]):
+        source = meta.get("source_file", "unknown")
+        chunk_idx = meta.get("chunk_index", "?")
+        page = meta.get("page_number")
+        if page is not None:
+            header = f"[source: {source}, page: {page}, chunk: {chunk_idx}]"
+        else:
+            header = f"[source: {source}, chunk: {chunk_idx}]"
+        chunks.append(f"{header}\n{doc}")
+
+    return chunks

tools/summarize_tool.py

@@ -0,0 +1,98 @@
+# COST: Gemini gemini-2.5-flash called ONCE per document on first summarization.
+# Result is written to a JSON sidecar file (<filename>.summary.json) beside
+# the source document.  Every subsequent call reads the sidecar and returns
+# immediately — zero additional tokens spent.
+# Sidecar is invalidated only by manual deletion.
+
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from ingestion import parse_document
+
+SUMMARY_SUFFIX = ".summary.json"
+MAX_SUMMARY_CHARS = 12_000  # truncation guard before sending to LLM
+
+
+def _cache_path(file_path: Path) -> Path:
+    return file_path.with_name(file_path.name + SUMMARY_SUFFIX)
+
+
+def summarize_document(file_path: str) -> str:
+    """Return a concise summary of a document, using a local cache when available.
+
+    On the first call the function extracts the full text from the document,
+    sends it to the OpenAI Chat Completions API (gpt-4o-mini), and writes the
+    resulting summary to a JSON sidecar file next to the original document
+    (e.g. 'report.pdf.summary.json').  Every subsequent call for the same file
+    reads the sidecar and returns immediately without making another LLM call,
+    keeping costs low and responses fast.
+
+    The cache is invalidated only by deleting the sidecar file manually.
+
+    Args:
+        file_path: Absolute or relative path to the .pdf or .docx document.
+
+    Returns:
+        A plain-text summary paragraph.  If the file cannot be found or parsed,
+        an error message is returned instead of raising so the agent can handle
+        it gracefully.
+    """
+    path = Path(file_path)
+
+    if not path.exists():
+        return f"Error: file not found at '{file_path}'."
+
+    cache = _cache_path(path)
+    if cache.exists():
+        try:
+            data = json.loads(cache.read_text(encoding="utf-8"))
+            return data["summary"]
+        except (json.JSONDecodeError, KeyError):
+            cache.unlink(missing_ok=True)  # corrupt cache — regenerate
+
+    try:
+        pages = parse_document(path)
+    except ValueError as exc:
+        return f"Error: {exc}"
+
+    full_text = "\n\n".join(p["text"] for p in pages)
+    truncated = full_text[:MAX_SUMMARY_CHARS]
+    if len(full_text) > MAX_SUMMARY_CHARS:
+        truncated += "\n\n[...document truncated for summarisation...]"
+
+    from openai import OpenAI
+
+    client = OpenAI(
+        api_key=os.environ["GEMINI_API_KEY"],
+        base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
+    )
+    response = client.chat.completions.create(
+        model="gemini-2.5-flash",
+        messages=[
+            {
+                "role": "system",
+                "content": (
+                    "You are a document summarisation assistant. "
+                    "Write a clear, concise summary (3-5 paragraphs) of the document "
+                    "the user provides. Focus on key topics, findings, and conclusions."
+                ),
+            },
+            {"role": "user", "content": f"Summarise this document:\n\n{truncated}"},
+        ],
+    )
+
+    summary = response.choices[0].message.content or ""
+    cache.write_text(
+        json.dumps({"source_file": path.name, "summary": summary}, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    return summary

tools/web_search_tool.py

@@ -0,0 +1,62 @@
+# COST: ZERO OpenAI tokens.
+# Retrieval uses BM25 keyword scoring (rank-bm25, runs locally).
+# Website content is read from the local scrape cache (content_store).
+# A live HTTP fetch only occurs when the cached snapshot is stale (>60 min).
+# No embedding model or LLM is invoked at any point in this tool.
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from rank_bm25 import BM25Okapi
+
+from scraper.content_store import is_content_stale, load_latest_content, save_site_content
+from scraper.web_scraper import AGENTRAX_URL, scrape_site
+
+_TOP_K = 5
+_MIN_CHARS = 40  # skip fragment-length paragraphs
+
+
+async def search_agentrax_website(query: str) -> str:
+    """Search the AgentRax website (https://agentrax.net/) for information relevant to the user query.
+
+    Use this tool when the user asks about AgentRax services, pricing, features, agents, or any
+    topic that would be found on the AgentRax website. This tool retrieves live or recently cached
+    website content — never answer AgentRax questions from general knowledge.
+
+    Returns: Relevant extracted sections from the website as a formatted string with source URL.
+    """
+    content = load_latest_content()
+
+    if is_content_stale(max_age_minutes=60) or not content:
+        fresh = await scrape_site(AGENTRAX_URL)
+        if "error" not in fresh:
+            save_site_content(fresh)
+            content = fresh
+
+    if not content:
+        return "Error: could not retrieve AgentRax website content."
+
+    body_text: str = content.get("body_text") or ""
+    title: str = content.get("title") or ""
+    url: str = content.get("url") or AGENTRAX_URL
+
+    paragraphs = [p.strip() for p in body_text.split("\n") if len(p.strip()) >= _MIN_CHARS]
+
+    if not paragraphs:
+        return f"Title: {title}\nRelevant Content:\n- (no content available)"
+
+    tokenized = [p.lower().split() for p in paragraphs]
+    bm25 = BM25Okapi(tokenized)
+    scores = bm25.get_scores(query.lower().split())
+
+    top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:_TOP_K]
+    top_paragraphs = [paragraphs[i] for i in sorted(top_indices)]
+
+    bullet_lines = "\n".join(f"- {p}" for p in top_paragraphs)
+    return f"Title: {title}\nRelevant Content:\n{bullet_lines}"