tools.py

# We build 2 custom tools using the @tool decorator. Firest the web_search tool which talks to the Tavily API and fetches live search results from the internet. Second the scrape_url tool which takes a URL, visits that page and extracts clean and readable text from it using BeautifulSoup.
from langchain.tools import tool
import requests
from bs4 import BeautifulSoup
from tavily import TavilyClient
import os
from dotenv import load_dotenv
from rich import print

load_dotenv()

tavily = TavilyClient(api_key=os.getenv("TAVILY_API_KEY"))
@tool
def web_search(query : str ) -> str:
    """Search the web for recent and reliable information on a given topic. Returns Titles, URLs and snippets """
    results = tavily.search(query= query, max_result=5)
    out=[]

    for r in results['results']:
        out.append(
            f'Title: {r["title"]}\nURL: {r["url"]}\nSnippet: {r["content"][:300]}\n'
        )
    return "\n----\n".join(out)
        
# print(web_search.invoke("What is the recent news of war"))


@tool
def scrape_url(url: str) -> str:
    """Scrape and return clean text content from a given URL."""
    try:
        resp= requests.get(url, timeout=8, headers= {"User-Agent": "Mozilla/5.0"})
        soup= BeautifulSoup(resp.text, "html.parser")
        for tag in soup(["script","style", "nav", "footer"]):
            tag.decompose()
        return soup.get_text(separator="\n", strip=True)[:3000]
    except Exception as e:
        return f"Could not scrape URL: {str(e)}"
    
# print(scrape_url.invoke("https://www.bbc.com/news/world-europe-66707497"))