import re from typing import List from urllib.parse import quote import markdownify as md import requests import wikipedia import wikipediaapi from bs4 import BeautifulSoup, Tag from langchain_core.tools import tool from pydantic import BaseModel class WikipediaPage(BaseModel): summary: str sections: List[str] @tool(parse_docstring=True) def find_wiki_page(query: str) -> str | None: """ Searches for a Wikipedia page based on the provided query and returns its summary and sections. This function searches for a Wikipedia page matching the query, retrieves the page content if found, and returns a JSON representation of the page's summary and section titles. Args: query (str): The search query to find a Wikipedia page. Returns: str | None: A JSON string containing the page's summary and section titles if found, or None if no matching page exists or if the search returns no results. Example: >>> result = find_wiki_page("Albert Einstein") >>> # Returns JSON with summary and sections of Einstein's Wikipedia page """ print(f"find_wiki_page called with query: {query}") # Search for the Wikipedia page page_titles = wikipedia.search(query, results=1) if not page_titles: print(f"No Wikipedia page found for the query '{query}'.") return None wiki = wikipediaapi.Wikipedia(user_agent="PythonWikiBot/1.0") page_title = page_titles[0] page = wiki.page(page_title) if not page.exists(): print(f"Page '{page_title}' does not exist.") return None def get_sections(section): sections = [] for sec in section.sections: sections.append(sec.title) sections.extend(get_sections(sec)) return sections result = WikipediaPage( summary=page.summary, sections=get_sections(page), ) return result.model_dump_json() def _get_heading_level(tag_or_class_name: str | Tag) -> int: """ Extract the heading level from a Tag object or a class name string. Args: tag_or_class_name: Either a BeautifulSoup Tag object or a string class name Returns: int: The heading level (1-6) or 0 if no valid heading level found """ # Handle both Tag objects and plain strings if isinstance(tag_or_class_name, Tag): classes = tag_or_class_name.attrs.get("class", []) else: classes = [tag_or_class_name] # Use regex to find mw-heading followed by a number pattern = re.compile(r"mw-heading(\d+)") for cls in classes: if match := pattern.search(cls): try: level = int(match.group(1)) return level if level > 0 else 0 except ValueError: continue return 0 def _compare_heading_levels(heading1: str | Tag, heading2: str | Tag) -> int: """ Compare the heading levels of two headings. Args: heading1: First heading (Tag object or class name string) heading2: Second heading (Tag object or class name string) Returns: int: Positive if heading1 > heading2, negative if heading1 < heading2, 0 if equal or if either heading level cannot be determined """ heading1_lvl = _get_heading_level(heading1) heading2_lvl = _get_heading_level(heading2) # Only compare if both heading levels are valid if heading1_lvl > 0 and heading2_lvl > 0: return heading1_lvl - heading2_lvl return 0 @tool(parse_docstring=True) def get_wiki_page_section_by_title(page_title: str, section_title: str) -> str | None: """ Retrieve the content of a specific section from a Wikipedia page. This function fetches a Wikipedia page by its title, then extracts the content of a specific section within that page based on the section title provided. The section content is converted from HTML to Markdown format. Args: page_title (str): The title of the Wikipedia page to fetch. section_title (str): The title of the section to extract from the page. Returns: str or None: The section content in Markdown format if found, otherwise None. """ print( f"get_wiki_page_section_by_title called with page_title: {page_title}, section_title: {section_title}" ) try: # Encode the page title for URL encoded_title = quote(page_title.replace(" ", "_")) url = f"https://en.wikipedia.org/wiki/{encoded_title}" # Fetch the page using requests headers = {"User-Agent": "PythonWikiBot/1.0"} response = requests.get(url, headers=headers) if response.status_code != 200: print( f"Failed to retrieve page '{page_title}'. Status code: {response.status_code}" ) return None page_html = response.text except Exception as e: print(f"Error fetching Wikipedia page '{page_title}': {e}") return None section_elements = [] soup = BeautifulSoup(page_html, "html.parser") # Find the section heading section_title_id = section_title.replace(" ", "_") section_heading = soup.find(attrs={"id": section_title_id}, recursive=True) if not section_heading or not isinstance(section_heading, Tag): print(f"Section '{section_title}' not found.") return None # Navigate to the parent div with class 'mw-heading' section_div = section_heading.parent while section_div: if section_div.has_attr("class"): if any(cls.startswith("mw-heading") for cls in section_div["class"]): break section_div = section_div.parent if not section_div: print(f"Section '{section_title}' not found.") return None # Extract all elements until the next section heading for element in section_div.find_next_siblings(): if not isinstance(element, Tag): continue if element.name == "span" and element.has_attr("class"): if "mw-editsection" in element["class"]: continue if element.name == "figure": continue if element.name == "div" and element.has_attr("class"): if _compare_heading_levels(element, section_div) <= 0: break section_elements.append(element) section_html = "".join(str(element) for element in section_elements) # Convert HTML to Markdown section_md = md.markdownify(section_html) print(f"Section content: {section_md}") return section_md