Andrei Sferdian commited on
Commit
ee405d5
·
1 Parent(s): 9d1cf8e

Add tools to search wikipedia

Browse files
.vscode/settings.json CHANGED
@@ -16,4 +16,5 @@
16
  ],
17
  "python.testing.unittestEnabled": false,
18
  "python.testing.pytestEnabled": true,
 
19
  }
 
16
  ],
17
  "python.testing.unittestEnabled": false,
18
  "python.testing.pytestEnabled": true,
19
+ "python.analysis.typeCheckingMode": "standard",
20
  }
agent.py CHANGED
@@ -2,12 +2,17 @@ from langchain_community.tools import DuckDuckGoSearchRun
2
  from langchain_together import ChatTogether
3
  from langgraph.prebuilt import create_react_agent
4
 
5
- from tools import GetYouTubeTranscript, search_vector_store
 
 
 
 
6
 
7
  tools = [
8
  DuckDuckGoSearchRun(),
9
  GetYouTubeTranscript(),
10
- search_vector_store,
 
11
  ]
12
 
13
  model = ChatTogether(
@@ -17,8 +22,18 @@ model = ChatTogether(
17
  max_retries=3,
18
  ).bind_tools(tools)
19
 
 
 
 
 
 
 
 
 
 
20
  agent = create_react_agent(
21
  model=model,
22
  tools=tools,
23
- # debug=True,
 
24
  )
 
2
  from langchain_together import ChatTogether
3
  from langgraph.prebuilt import create_react_agent
4
 
5
+ from tools import GetYouTubeTranscript
6
+ from tools.wikipedia import (
7
+ find_wiki_page,
8
+ get_wiki_page_section_by_title,
9
+ )
10
 
11
  tools = [
12
  DuckDuckGoSearchRun(),
13
  GetYouTubeTranscript(),
14
+ find_wiki_page,
15
+ get_wiki_page_section_by_title,
16
  ]
17
 
18
  model = ChatTogether(
 
22
  max_retries=3,
23
  ).bind_tools(tools)
24
 
25
+ prompt = (
26
+ "You are a helpful assistant that answers questions using Wikipedia.\n"
27
+ "When searching for Wikipedia content, always follow this specific sequence:\n"
28
+ " a. Use find_wiki_page tool to locate the relevant Wikipedia page.\n"
29
+ " The response will be a JSON string with the page's summary and sections.\n"
30
+ " b. If the page exists, and the summary does not contain the required info, use get_wiki_page_section_by_title tool to get the content of the desired section.\n"
31
+ "If the user asks for a YouTube transcript, use the GetYouTubeTranscript tool.\n"
32
+ )
33
+
34
  agent = create_react_agent(
35
  model=model,
36
  tools=tools,
37
+ prompt=prompt,
38
+ debug=True,
39
  )
app.py CHANGED
@@ -9,7 +9,7 @@ from langchain_core.messages import HumanMessage
9
 
10
  from agent import agent
11
  from langfuse_handler import create_langfuse_handler
12
- from questions import Answer, Question, get_questions
13
 
14
  # (Keep Constants as is)
15
  # --- Constants ---
@@ -254,20 +254,32 @@ def run_in_space():
254
  def run_locally():
255
  print("Running locally...")
256
 
257
- questions = get_questions(1)
258
- if not questions:
259
- print("No questions available to run.")
260
- return
 
 
 
 
 
 
 
 
 
261
 
262
  max_concurrency = int(os.getenv("MAX_CONCURRENCY", 1))
263
  with ThreadPoolExecutor(max_workers=max_concurrency) as executor:
264
  answers = list(executor.map(process_question, questions))
265
 
 
 
266
 
267
  def process_question(question: Question) -> Answer:
268
  task_id = question.task_id
269
  question_text = question.question
270
  answer = Answer(task_id=task_id, answer="")
 
271
  print(f"Processing question: {question}")
272
  try:
273
  response = agent.invoke(
 
9
 
10
  from agent import agent
11
  from langfuse_handler import create_langfuse_handler
12
+ from questions import Answer, Question
13
 
14
  # (Keep Constants as is)
15
  # --- Constants ---
 
254
  def run_locally():
255
  print("Running locally...")
256
 
257
+ # questions = get_questions(1)
258
+ # if not questions:
259
+ # print("No questions available to run.")
260
+ # return
261
+ questions = [
262
+ Question(
263
+ **{
264
+ "task_id": "8e867cd7-cff9-4e6c-867a-ff5ddc2550be",
265
+ "question": "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia.",
266
+ "file_name": "",
267
+ }
268
+ )
269
+ ]
270
 
271
  max_concurrency = int(os.getenv("MAX_CONCURRENCY", 1))
272
  with ThreadPoolExecutor(max_workers=max_concurrency) as executor:
273
  answers = list(executor.map(process_question, questions))
274
 
275
+ print(answers)
276
+
277
 
278
  def process_question(question: Question) -> Answer:
279
  task_id = question.task_id
280
  question_text = question.question
281
  answer = Answer(task_id=task_id, answer="")
282
+
283
  print(f"Processing question: {question}")
284
  try:
285
  response = agent.invoke(
requirements.txt CHANGED
@@ -191,6 +191,7 @@ uvicorn==0.34.2
191
  webencodings==0.5.1
192
  websockets==15.0.1
193
  wikipedia==1.4.0
 
194
  wrapt==1.17.2
195
  xlrd==2.0.1
196
  XlsxWriter==3.2.3
 
191
  webencodings==0.5.1
192
  websockets==15.0.1
193
  wikipedia==1.4.0
194
+ Wikipedia-API==0.8.1
195
  wrapt==1.17.2
196
  xlrd==2.0.1
197
  XlsxWriter==3.2.3
tools/__init__.py CHANGED
@@ -1,7 +1,11 @@
1
  from .get_youtube_transcript import GetYouTubeTranscript
2
- from .search_vector_store import search_vector_store
 
 
 
3
 
4
  __all__ = [
5
- "search_vector_store",
6
  "GetYouTubeTranscript",
 
 
7
  ]
 
1
  from .get_youtube_transcript import GetYouTubeTranscript
2
+ from .wikipedia import (
3
+ find_wiki_page,
4
+ get_wiki_page_section_by_title,
5
+ )
6
 
7
  __all__ = [
 
8
  "GetYouTubeTranscript",
9
+ "find_wiki_page",
10
+ "get_wiki_page_section_by_title",
11
  ]
tools/search_vector_store.py DELETED
@@ -1,62 +0,0 @@
1
- import wikipedia as wiki
2
- from bs4 import BeautifulSoup
3
- from langchain_core.tools import tool
4
- from langchain_core.vectorstores import InMemoryVectorStore
5
- from langchain_text_splitters import HTMLHeaderTextSplitter
6
- from langchain_together import TogetherEmbeddings
7
-
8
- vector_store = InMemoryVectorStore(TogetherEmbeddings())
9
- retriever = vector_store.as_retriever()
10
- text_splitter = HTMLHeaderTextSplitter(
11
- headers_to_split_on=[("h2", "Section"), ("h3", "Subsection")],
12
- )
13
-
14
-
15
- @tool(description="Search the vector store for data matching the given query.")
16
- def search_vector_store(query: str) -> str:
17
- """
18
- Searches the vector store for data matching the given query.
19
-
20
- Args:
21
- query (str): The search query to look up on Wikipedia.
22
-
23
- Returns:
24
- str: The concatenated content of the retrieved documents or an error message if no
25
- relevant page is found.
26
-
27
- Raises:
28
- Exception: Raised if there is any error while fetching data
29
- """
30
- try:
31
- print(f"Searching Wikipedia for: {query}")
32
- titles = wiki.search(query)
33
- if not titles or len(titles) == 0:
34
- return "No Wikipedia page found for the given query."
35
- print(f"Found titles: {titles}")
36
-
37
- # Fetch the Wikipedia page
38
- print(f"Fetching Wikipedia page for title: {titles[0]}")
39
- page = wiki.page(titles[0])
40
-
41
- # Parse the HTML content
42
- soup = BeautifulSoup(page.html(), "html.parser")
43
- content_div = soup.find(id="mw-content-text")
44
-
45
- section = page.section("kek")
46
-
47
- print("Splitting content...")
48
- docs = text_splitter.split_text(content_div.decode())
49
-
50
- print("Adding documents to vector store...")
51
- doc_ids = vector_store.add_documents(docs)
52
- print(f"Added {len(doc_ids)} documents to the vector store.")
53
-
54
- print("Retrieving documents from vector store...")
55
- retriever.invoke(input=query)
56
- print(f"Retrieved {len(docs)} documents from the vector store.")
57
- result = "\n".join([doc.page_content for doc in docs])
58
- print(f"Result: {result}")
59
- return result
60
-
61
- except Exception as e:
62
- raise Exception(f"Error fetching data: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tools/wikipedia.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from typing import List
3
+
4
+ import markdownify as md
5
+ import wikipedia
6
+ import wikipedia as wiki
7
+ import wikipediaapi
8
+ from bs4 import BeautifulSoup, Tag
9
+ from langchain_core.tools import tool
10
+ from pydantic import BaseModel
11
+
12
+
13
+ class WikipediaPage(BaseModel):
14
+ summary: str
15
+ sections: List[str]
16
+
17
+
18
+ @tool()
19
+ def find_wiki_page(query: str) -> str | None:
20
+ """
21
+ Searches for a Wikipedia page based on the provided query and returns its summary and sections.
22
+
23
+ This function searches for a Wikipedia page matching the query, retrieves the page content
24
+ if found, and returns a JSON representation of the page's summary and section titles.
25
+
26
+ Args:
27
+ query (str): The search query to find a Wikipedia page.
28
+
29
+ Returns:
30
+ str | None: A JSON string containing the page's summary and section titles if found,
31
+ or None if no matching page exists or if the search returns no results.
32
+
33
+ Example:
34
+ >>> result = find_wiki_page("Albert Einstein")
35
+ >>> # Returns JSON with summary and sections of Einstein's Wikipedia page
36
+ """
37
+ print(f"find_wiki_page called with query: {query}")
38
+
39
+ # Search for the Wikipedia page
40
+ page_titles = wikipedia.search(query, results=1)
41
+ if not page_titles:
42
+ print(f"No Wikipedia page found for the query '{query}'.")
43
+ return None
44
+
45
+ wiki = wikipediaapi.Wikipedia(user_agent="PythonWikiBot/1.0")
46
+ page_title = page_titles[0]
47
+ page = wiki.page(page_title)
48
+ if not page.exists():
49
+ print(f"Page '{page_title}' does not exist.")
50
+ return None
51
+
52
+ result = WikipediaPage(
53
+ summary=page.summary,
54
+ sections=[section.title for section in page.sections],
55
+ )
56
+
57
+ return result.model_dump_json()
58
+
59
+
60
+ @tool(
61
+ description=(
62
+ "Get the content of a specific section by title from a Wikipedia page.\n"
63
+ "Input: Two strings - the first is the title of the Wikipedia page (e.g., 'Python (programming language)'), "
64
+ "and the second is the title of the section (e.g., 'History').\n"
65
+ "Output: A string containing the content of the specified section in Markdown format, or None if the section does not exist."
66
+ )
67
+ )
68
+ def get_wiki_page_section_by_title(page_title: str, section_title: str) -> str | None:
69
+ """
70
+ Get the content of a specific section by title.
71
+ """
72
+ print(
73
+ f"get_wiki_page_section_by_title called with page_title: {page_title}, section_title: {section_title}"
74
+ )
75
+ page = wiki.page(page_title)
76
+ if not page:
77
+ print(f"Page '{page_title}' does not exist.")
78
+ return None
79
+
80
+ section_elements = []
81
+ soup = BeautifulSoup(page.html(), "html.parser")
82
+ # Find the section heading
83
+ section_heading = soup.find(id=section_title)
84
+ if not section_heading:
85
+ print(f"Section '{section_title}' not found.")
86
+ return None
87
+
88
+ # Navigate to the parent div with class 'mw-heading'
89
+ parent = section_heading.parent
90
+ while parent:
91
+ if parent.has_attr("class"):
92
+ if any(cls.startswith("mw-heading") for cls in parent["class"]):
93
+ break
94
+ parent = parent.parent
95
+
96
+ if not parent:
97
+ print(f"Section '{section_title}' not found.")
98
+ return None
99
+
100
+ next_section_class = None
101
+ for cls in parent["class"]:
102
+ pattern = re.compile(r"mw-heading\d+")
103
+ match = pattern.search(cls)
104
+ if match:
105
+ next_section_class = match.group()
106
+ break
107
+
108
+ if not next_section_class:
109
+ print(f"Next section class not found for '{section_title}'.")
110
+ return None
111
+
112
+ # Extract all elements until the next section heading
113
+ for element in parent.find_next_siblings():
114
+ if not isinstance(element, Tag):
115
+ continue
116
+
117
+ if element.name == "span" and element.has_attr("class"):
118
+ if "mw-editsection" in element["class"]:
119
+ continue
120
+
121
+ if element.name == "figure":
122
+ continue
123
+
124
+ if element.name == "div" and element.has_attr("class"):
125
+ if any(cls.startswith(next_section_class) for cls in element["class"]):
126
+ break
127
+
128
+ section_elements.append(element)
129
+
130
+ section_html = "".join(str(element) for element in section_elements)
131
+
132
+ # Convert HTML to Markdown
133
+ section_md = md.markdownify(section_html)
134
+ print(f"Section content: {section_md}")
135
+ return section_md