Download tools/wikipedia.py from andrei-sferdian/Agents_Course_Final_Assignment: direct link, hf CLI and curl.
- Browser
- Download file 6.61 kB
-
https://huggingface.co/spaces/andrei-sferdian/Agents_Course_Final_Assignment/resolve/main/tools/wikipedia.py
- Command line
-
hf download hf://spaces/andrei-sferdian/Agents_Course_Final_Assignment/tools/wikipedia.py
-
curl -L -o wikipedia.py https://huggingface.co/spaces/andrei-sferdian/Agents_Course_Final_Assignment/resolve/main/tools/wikipedia.py
6.61 kB
| import re | |
| from typing import List | |
| from urllib.parse import quote | |
| import markdownify as md | |
| import requests | |
| import wikipedia | |
| import wikipediaapi | |
| from bs4 import BeautifulSoup, Tag | |
| from langchain_core.tools import tool | |
| from pydantic import BaseModel | |
| class WikipediaPage(BaseModel): | |
| summary: str | |
| sections: List[str] | |
| def find_wiki_page(query: str) -> str | None: | |
| """ | |
| Searches for a Wikipedia page based on the provided query and returns its summary and sections. | |
| This function searches for a Wikipedia page matching the query, retrieves the page content | |
| if found, and returns a JSON representation of the page's summary and section titles. | |
| Args: | |
| query (str): The search query to find a Wikipedia page. | |
| Returns: | |
| str | None: A JSON string containing the page's summary and section titles if found, | |
| or None if no matching page exists or if the search returns no results. | |
| Example: | |
| >>> result = find_wiki_page("Albert Einstein") | |
| >>> # Returns JSON with summary and sections of Einstein's Wikipedia page | |
| """ | |
| print(f"find_wiki_page called with query: {query}") | |
| # Search for the Wikipedia page | |
| page_titles = wikipedia.search(query, results=1) | |
| if not page_titles: | |
| print(f"No Wikipedia page found for the query '{query}'.") | |
| return None | |
| wiki = wikipediaapi.Wikipedia(user_agent="PythonWikiBot/1.0") | |
| page_title = page_titles[0] | |
| page = wiki.page(page_title) | |
| if not page.exists(): | |
| print(f"Page '{page_title}' does not exist.") | |
| return None | |
| def get_sections(section): | |
| sections = [] | |
| for sec in section.sections: | |
| sections.append(sec.title) | |
| sections.extend(get_sections(sec)) | |
| return sections | |
| result = WikipediaPage( | |
| summary=page.summary, | |
| sections=get_sections(page), | |
| ) | |
| return result.model_dump_json() | |
| def _get_heading_level(tag_or_class_name: str | Tag) -> int: | |
| """ | |
| Extract the heading level from a Tag object or a class name string. | |
| Args: | |
| tag_or_class_name: Either a BeautifulSoup Tag object or a string class name | |
| Returns: | |
| int: The heading level (1-6) or 0 if no valid heading level found | |
| """ | |
| # Handle both Tag objects and plain strings | |
| if isinstance(tag_or_class_name, Tag): | |
| classes = tag_or_class_name.attrs.get("class", []) | |
| else: | |
| classes = [tag_or_class_name] | |
| # Use regex to find mw-heading followed by a number | |
| pattern = re.compile(r"mw-heading(\d+)") | |
| for cls in classes: | |
| if match := pattern.search(cls): | |
| try: | |
| level = int(match.group(1)) | |
| return level if level > 0 else 0 | |
| except ValueError: | |
| continue | |
| return 0 | |
| def _compare_heading_levels(heading1: str | Tag, heading2: str | Tag) -> int: | |
| """ | |
| Compare the heading levels of two headings. | |
| Args: | |
| heading1: First heading (Tag object or class name string) | |
| heading2: Second heading (Tag object or class name string) | |
| Returns: | |
| int: Positive if heading1 > heading2, negative if heading1 < heading2, | |
| 0 if equal or if either heading level cannot be determined | |
| """ | |
| heading1_lvl = _get_heading_level(heading1) | |
| heading2_lvl = _get_heading_level(heading2) | |
| # Only compare if both heading levels are valid | |
| if heading1_lvl > 0 and heading2_lvl > 0: | |
| return heading1_lvl - heading2_lvl | |
| return 0 | |
| def get_wiki_page_section_by_title(page_title: str, section_title: str) -> str | None: | |
| """ | |
| Retrieve the content of a specific section from a Wikipedia page. | |
| This function fetches a Wikipedia page by its title, then extracts the content | |
| of a specific section within that page based on the section title provided. | |
| The section content is converted from HTML to Markdown format. | |
| Args: | |
| page_title (str): The title of the Wikipedia page to fetch. | |
| section_title (str): The title of the section to extract from the page. | |
| Returns: | |
| str or None: The section content in Markdown format if found, otherwise None. | |
| """ | |
| print( | |
| f"get_wiki_page_section_by_title called with page_title: {page_title}, section_title: {section_title}" | |
| ) | |
| try: | |
| # Encode the page title for URL | |
| encoded_title = quote(page_title.replace(" ", "_")) | |
| url = f"https://en.wikipedia.org/wiki/{encoded_title}" | |
| # Fetch the page using requests | |
| headers = {"User-Agent": "PythonWikiBot/1.0"} | |
| response = requests.get(url, headers=headers) | |
| if response.status_code != 200: | |
| print( | |
| f"Failed to retrieve page '{page_title}'. Status code: {response.status_code}" | |
| ) | |
| return None | |
| page_html = response.text | |
| except Exception as e: | |
| print(f"Error fetching Wikipedia page '{page_title}': {e}") | |
| return None | |
| section_elements = [] | |
| soup = BeautifulSoup(page_html, "html.parser") | |
| # Find the section heading | |
| section_title_id = section_title.replace(" ", "_") | |
| section_heading = soup.find(attrs={"id": section_title_id}, recursive=True) | |
| if not section_heading or not isinstance(section_heading, Tag): | |
| print(f"Section '{section_title}' not found.") | |
| return None | |
| # Navigate to the parent div with class 'mw-heading' | |
| section_div = section_heading.parent | |
| while section_div: | |
| if section_div.has_attr("class"): | |
| if any(cls.startswith("mw-heading") for cls in section_div["class"]): | |
| break | |
| section_div = section_div.parent | |
| if not section_div: | |
| print(f"Section '{section_title}' not found.") | |
| return None | |
| # Extract all elements until the next section heading | |
| for element in section_div.find_next_siblings(): | |
| if not isinstance(element, Tag): | |
| continue | |
| if element.name == "span" and element.has_attr("class"): | |
| if "mw-editsection" in element["class"]: | |
| continue | |
| if element.name == "figure": | |
| continue | |
| if element.name == "div" and element.has_attr("class"): | |
| if _compare_heading_levels(element, section_div) <= 0: | |
| break | |
| section_elements.append(element) | |
| section_html = "".join(str(element) for element in section_elements) | |
| # Convert HTML to Markdown | |
| section_md = md.markdownify(section_html) | |
| print(f"Section content: {section_md}") | |
| return section_md | |