Andrei Sferdian
Use gpt-4.1 as model
9b0b7f6
Raw History Blame Contribute Delete
6.61 kB
import re
from typing import List
from urllib.parse import quote
import markdownify as md
import requests
import wikipedia
import wikipediaapi
from bs4 import BeautifulSoup, Tag
from langchain_core.tools import tool
from pydantic import BaseModel
class WikipediaPage(BaseModel):
summary: str
sections: List[str]
@tool(parse_docstring=True)
def find_wiki_page(query: str) -> str | None:
"""
Searches for a Wikipedia page based on the provided query and returns its summary and sections.
This function searches for a Wikipedia page matching the query, retrieves the page content
if found, and returns a JSON representation of the page's summary and section titles.
Args:
query (str): The search query to find a Wikipedia page.
Returns:
str | None: A JSON string containing the page's summary and section titles if found,
or None if no matching page exists or if the search returns no results.
Example:
>>> result = find_wiki_page("Albert Einstein")
>>> # Returns JSON with summary and sections of Einstein's Wikipedia page
"""
print(f"find_wiki_page called with query: {query}")
# Search for the Wikipedia page
page_titles = wikipedia.search(query, results=1)
if not page_titles:
print(f"No Wikipedia page found for the query '{query}'.")
return None
wiki = wikipediaapi.Wikipedia(user_agent="PythonWikiBot/1.0")
page_title = page_titles[0]
page = wiki.page(page_title)
if not page.exists():
print(f"Page '{page_title}' does not exist.")
return None
def get_sections(section):
sections = []
for sec in section.sections:
sections.append(sec.title)
sections.extend(get_sections(sec))
return sections
result = WikipediaPage(
summary=page.summary,
sections=get_sections(page),
)
return result.model_dump_json()
def _get_heading_level(tag_or_class_name: str | Tag) -> int:
"""
Extract the heading level from a Tag object or a class name string.
Args:
tag_or_class_name: Either a BeautifulSoup Tag object or a string class name
Returns:
int: The heading level (1-6) or 0 if no valid heading level found
"""
# Handle both Tag objects and plain strings
if isinstance(tag_or_class_name, Tag):
classes = tag_or_class_name.attrs.get("class", [])
else:
classes = [tag_or_class_name]
# Use regex to find mw-heading followed by a number
pattern = re.compile(r"mw-heading(\d+)")
for cls in classes:
if match := pattern.search(cls):
try:
level = int(match.group(1))
return level if level > 0 else 0
except ValueError:
continue
return 0
def _compare_heading_levels(heading1: str | Tag, heading2: str | Tag) -> int:
"""
Compare the heading levels of two headings.
Args:
heading1: First heading (Tag object or class name string)
heading2: Second heading (Tag object or class name string)
Returns:
int: Positive if heading1 > heading2, negative if heading1 < heading2,
0 if equal or if either heading level cannot be determined
"""
heading1_lvl = _get_heading_level(heading1)
heading2_lvl = _get_heading_level(heading2)
# Only compare if both heading levels are valid
if heading1_lvl > 0 and heading2_lvl > 0:
return heading1_lvl - heading2_lvl
return 0
@tool(parse_docstring=True)
def get_wiki_page_section_by_title(page_title: str, section_title: str) -> str | None:
"""
Retrieve the content of a specific section from a Wikipedia page.
This function fetches a Wikipedia page by its title, then extracts the content
of a specific section within that page based on the section title provided.
The section content is converted from HTML to Markdown format.
Args:
page_title (str): The title of the Wikipedia page to fetch.
section_title (str): The title of the section to extract from the page.
Returns:
str or None: The section content in Markdown format if found, otherwise None.
"""
print(
f"get_wiki_page_section_by_title called with page_title: {page_title}, section_title: {section_title}"
)
try:
# Encode the page title for URL
encoded_title = quote(page_title.replace(" ", "_"))
url = f"https://en.wikipedia.org/wiki/{encoded_title}"
# Fetch the page using requests
headers = {"User-Agent": "PythonWikiBot/1.0"}
response = requests.get(url, headers=headers)
if response.status_code != 200:
print(
f"Failed to retrieve page '{page_title}'. Status code: {response.status_code}"
)
return None
page_html = response.text
except Exception as e:
print(f"Error fetching Wikipedia page '{page_title}': {e}")
return None
section_elements = []
soup = BeautifulSoup(page_html, "html.parser")
# Find the section heading
section_title_id = section_title.replace(" ", "_")
section_heading = soup.find(attrs={"id": section_title_id}, recursive=True)
if not section_heading or not isinstance(section_heading, Tag):
print(f"Section '{section_title}' not found.")
return None
# Navigate to the parent div with class 'mw-heading'
section_div = section_heading.parent
while section_div:
if section_div.has_attr("class"):
if any(cls.startswith("mw-heading") for cls in section_div["class"]):
break
section_div = section_div.parent
if not section_div:
print(f"Section '{section_title}' not found.")
return None
# Extract all elements until the next section heading
for element in section_div.find_next_siblings():
if not isinstance(element, Tag):
continue
if element.name == "span" and element.has_attr("class"):
if "mw-editsection" in element["class"]:
continue
if element.name == "figure":
continue
if element.name == "div" and element.has_attr("class"):
if _compare_heading_levels(element, section_div) <= 0:
break
section_elements.append(element)
section_html = "".join(str(element) for element in section_elements)
# Convert HTML to Markdown
section_md = md.markdownify(section_html)
print(f"Section content: {section_md}")
return section_md