Andrei Sferdian commited on
Commit
331d85e
·
1 Parent(s): ebe102b

Add vector store module

Browse files
agent.py CHANGED
@@ -36,15 +36,15 @@ builder = StateGraph(state_schema=AgentState)
36
 
37
 
38
  class Urls:
39
- BASE_URL = "https://agents-course-unit4-scoring.hf.space"
40
 
41
  @classmethod
42
  def questions(cls) -> str:
43
- return f"{cls.BASE_URL}/questions"
44
 
45
  @classmethod
46
  def files(cls, task_id: str) -> str:
47
- return f"{cls.BASE_URL}/files/{task_id}"
48
 
49
 
50
  if __name__ == "__main__":
 
36
 
37
 
38
  class Urls:
39
+ TASKS_URL = "https://agents-course-unit4-scoring.hf.space"
40
 
41
  @classmethod
42
  def questions(cls) -> str:
43
+ return f"{cls.TASKS_URL}/questions"
44
 
45
  @classmethod
46
  def files(cls, task_id: str) -> str:
47
+ return f"{cls.TASKS_URL}/files/{task_id}"
48
 
49
 
50
  if __name__ == "__main__":
questions.py CHANGED
@@ -12,9 +12,9 @@ class Question(BaseModel):
12
 
13
 
14
  def get_questions(limit: int = None) -> list[Question]:
15
- base_url = os.getenv("BASE_URL")
16
  if base_url is None:
17
- raise ValueError("BASE_URL environment variable is not set")
18
 
19
  try:
20
  response = requests.get(base_url + "/questions", timeout=15)
 
12
 
13
 
14
  def get_questions(limit: int = None) -> list[Question]:
15
+ base_url = os.getenv("TASKS_URL")
16
  if base_url is None:
17
+ raise ValueError("TASKS_URL environment variable is not set")
18
 
19
  try:
20
  response = requests.get(base_url + "/questions", timeout=15)
tools/__init__.py CHANGED
@@ -1,7 +1,7 @@
1
- from .get_wikipedia_page import get_wikipedia_page
2
  from .get_youtube_transcript import GetYouTubeTranscript
 
3
 
4
  __all__ = [
5
- "get_wikipedia_page",
6
  "GetYouTubeTranscript",
7
  ]
 
 
1
  from .get_youtube_transcript import GetYouTubeTranscript
2
+ from .search_vector_store import search_vector_store
3
 
4
  __all__ = [
5
+ "search_vector_store",
6
  "GetYouTubeTranscript",
7
  ]
tools/get_wikipedia_page.py DELETED
@@ -1,33 +0,0 @@
1
- import requests
2
- import wikipedia as wiki
3
- from langchain_core.tools import tool
4
-
5
-
6
- @tool(description="Get Wikipedia page content")
7
- def get_wikipedia_page(query: str) -> str:
8
- """
9
- Fetches the content of a Wikipedia page based on the given query.
10
- This function searches for Wikipedia pages matching the query, retrieves the first result,
11
- and fetches its content. If no pages are found or an error occurs during the process,
12
- an appropriate message or error description is returned.
13
- Args:
14
- query (str): The search query to look for on Wikipedia.
15
- Returns:
16
- str: The content of the Wikipedia page if found and successfully fetched,
17
- or an error message if no page is found or an exception occurs.
18
- Raises:
19
- Exception: If an error occurs while fetching the Wikipedia page.
20
- """
21
-
22
- try:
23
- titles = wiki.search(query)
24
- if not titles or len(titles) == 0:
25
- return "No Wikipedia page found for the given query."
26
-
27
- page = wiki.page(titles[0])
28
- url = page.url
29
- response = requests.get(url)
30
- response.raise_for_status()
31
- return response.text
32
- except Exception as e:
33
- raise Exception(f"Error fetching Wikipedia page: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tools/search_vector_store.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import wikipedia as wiki
2
+ from bs4 import BeautifulSoup
3
+ from langchain_core.tools import tool
4
+ from langchain_core.vectorstores import InMemoryVectorStore
5
+ from langchain_text_splitters import HTMLHeaderTextSplitter
6
+ from langchain_together import TogetherEmbeddings
7
+
8
+ vector_store = InMemoryVectorStore(TogetherEmbeddings())
9
+ retriever = vector_store.as_retriever()
10
+ text_splitter = HTMLHeaderTextSplitter(
11
+ headers_to_split_on=[("h2", "Section"), ("h3", "Subsection")],
12
+ )
13
+
14
+
15
+ @tool(description="Search the vector store for data matching the given query.")
16
+ def search_vector_store(query: str) -> str:
17
+ """
18
+ Searches the vector store for data matching the given query.
19
+
20
+ Args:
21
+ query (str): The search query to look up on Wikipedia.
22
+
23
+ Returns:
24
+ str: The concatenated content of the retrieved documents or an error message if no
25
+ relevant page is found.
26
+
27
+ Raises:
28
+ Exception: Raised if there is any error while fetching data
29
+ """
30
+ try:
31
+ print(f"Searching Wikipedia for: {query}")
32
+ titles = wiki.search(query)
33
+ if not titles or len(titles) == 0:
34
+ return "No Wikipedia page found for the given query."
35
+ print(f"Found titles: {titles}")
36
+
37
+ # Fetch the Wikipedia page
38
+ print(f"Fetching Wikipedia page for title: {titles[0]}")
39
+ page = wiki.page(titles[0])
40
+
41
+ # Parse the HTML content
42
+ soup = BeautifulSoup(page.html(), "html.parser")
43
+ content_div = soup.find(id="mw-content-text")
44
+
45
+ section = page.section("kek")
46
+
47
+ print("Splitting content...")
48
+ docs = text_splitter.split_text(content_div.decode())
49
+
50
+ print("Adding documents to vector store...")
51
+ doc_ids = vector_store.add_documents(docs)
52
+ print(f"Added {len(doc_ids)} documents to the vector store.")
53
+
54
+ print("Retrieving documents from vector store...")
55
+ retriever.invoke(input=query)
56
+ print(f"Retrieved {len(docs)} documents from the vector store.")
57
+ result = "\n".join([doc.page_content for doc in docs])
58
+ print(f"Result: {result}")
59
+ return result
60
+
61
+ except Exception as e:
62
+ raise Exception(f"Error fetching data: {e}")
vector_store.py DELETED
@@ -1,58 +0,0 @@
1
- from langchain_core.documents import Document
2
- from langchain_core.messages import SystemMessage
3
- from langchain_core.pydantic_v1 import BaseModel, Field
4
- from langchain_core.vectorstores.in_memory import InMemoryVectorStore
5
- from langchain_together import ChatTogether
6
- from langchain_together.embeddings import TogetherEmbeddings
7
-
8
- from questions import get_questions
9
- from tools import get_wikipedia_page
10
-
11
-
12
- class ResponseSchema(BaseModel):
13
- search_query: str | None = Field(
14
- ...,
15
- description="The search query to be used for Wikipedia search, or None if no search is needed.",
16
- )
17
-
18
-
19
- if __name__ == "__main__":
20
- vector_store = InMemoryVectorStore(TogetherEmbeddings())
21
- model = model = ChatTogether(
22
- model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
23
- temperature=0,
24
- max_retries=3,
25
- timeout=15,
26
- ).with_structured_output(schema=ResponseSchema, method="json_schema", strict=True)
27
- questions = get_questions(limit=3)
28
- print(f"Fetched {len(questions)} questions.")
29
-
30
- sys_msg = SystemMessage(
31
- content=(
32
- "You are an intelligent and resourceful assistant. "
33
- "Your task is to determine if a Wikipedia search is necessary to provide an accurate and comprehensive answer to the given question. "
34
- ),
35
- )
36
- for question in questions:
37
- print(f"Question: {question.question}")
38
- response: ResponseSchema = model.invoke([sys_msg, question.question])
39
-
40
- if response.search_query is None:
41
- print("No search query needed.")
42
- continue
43
-
44
- print(f"Search query: {response.search_query}")
45
- wikipedia_page = get_wikipedia_page(response.search_query)
46
- doc = Document(page_content=wikipedia_page)
47
- vector_store.add_documents([doc])
48
-
49
- while True:
50
- user_input = input("Enter your message (or 'exit' to quit): ")
51
- if user_input.lower() in ["exit", "quit", "q"]:
52
- break
53
-
54
- # Search the vector store for relevant documents
55
- docs = vector_store.similarity_search(user_input)
56
- print(f"Found {len(docs)} documents in the vector store.")
57
- for doc in docs:
58
- print(f"Document: {doc.page_content}")