Andrei Sferdian commited on
Commit ·
331d85e
1
Parent(s): ebe102b
Add vector store module
Browse files- agent.py +3 -3
- questions.py +2 -2
- tools/__init__.py +2 -2
- tools/get_wikipedia_page.py +0 -33
- tools/search_vector_store.py +62 -0
- vector_store.py +0 -58
agent.py
CHANGED
|
@@ -36,15 +36,15 @@ builder = StateGraph(state_schema=AgentState)
|
|
| 36 |
|
| 37 |
|
| 38 |
class Urls:
|
| 39 |
-
|
| 40 |
|
| 41 |
@classmethod
|
| 42 |
def questions(cls) -> str:
|
| 43 |
-
return f"{cls.
|
| 44 |
|
| 45 |
@classmethod
|
| 46 |
def files(cls, task_id: str) -> str:
|
| 47 |
-
return f"{cls.
|
| 48 |
|
| 49 |
|
| 50 |
if __name__ == "__main__":
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
class Urls:
|
| 39 |
+
TASKS_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 40 |
|
| 41 |
@classmethod
|
| 42 |
def questions(cls) -> str:
|
| 43 |
+
return f"{cls.TASKS_URL}/questions"
|
| 44 |
|
| 45 |
@classmethod
|
| 46 |
def files(cls, task_id: str) -> str:
|
| 47 |
+
return f"{cls.TASKS_URL}/files/{task_id}"
|
| 48 |
|
| 49 |
|
| 50 |
if __name__ == "__main__":
|
questions.py
CHANGED
|
@@ -12,9 +12,9 @@ class Question(BaseModel):
|
|
| 12 |
|
| 13 |
|
| 14 |
def get_questions(limit: int = None) -> list[Question]:
|
| 15 |
-
base_url = os.getenv("
|
| 16 |
if base_url is None:
|
| 17 |
-
raise ValueError("
|
| 18 |
|
| 19 |
try:
|
| 20 |
response = requests.get(base_url + "/questions", timeout=15)
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
def get_questions(limit: int = None) -> list[Question]:
|
| 15 |
+
base_url = os.getenv("TASKS_URL")
|
| 16 |
if base_url is None:
|
| 17 |
+
raise ValueError("TASKS_URL environment variable is not set")
|
| 18 |
|
| 19 |
try:
|
| 20 |
response = requests.get(base_url + "/questions", timeout=15)
|
tools/__init__.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
-
from .get_wikipedia_page import get_wikipedia_page
|
| 2 |
from .get_youtube_transcript import GetYouTubeTranscript
|
|
|
|
| 3 |
|
| 4 |
__all__ = [
|
| 5 |
-
"
|
| 6 |
"GetYouTubeTranscript",
|
| 7 |
]
|
|
|
|
|
|
|
| 1 |
from .get_youtube_transcript import GetYouTubeTranscript
|
| 2 |
+
from .search_vector_store import search_vector_store
|
| 3 |
|
| 4 |
__all__ = [
|
| 5 |
+
"search_vector_store",
|
| 6 |
"GetYouTubeTranscript",
|
| 7 |
]
|
tools/get_wikipedia_page.py
DELETED
|
@@ -1,33 +0,0 @@
|
|
| 1 |
-
import requests
|
| 2 |
-
import wikipedia as wiki
|
| 3 |
-
from langchain_core.tools import tool
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
@tool(description="Get Wikipedia page content")
|
| 7 |
-
def get_wikipedia_page(query: str) -> str:
|
| 8 |
-
"""
|
| 9 |
-
Fetches the content of a Wikipedia page based on the given query.
|
| 10 |
-
This function searches for Wikipedia pages matching the query, retrieves the first result,
|
| 11 |
-
and fetches its content. If no pages are found or an error occurs during the process,
|
| 12 |
-
an appropriate message or error description is returned.
|
| 13 |
-
Args:
|
| 14 |
-
query (str): The search query to look for on Wikipedia.
|
| 15 |
-
Returns:
|
| 16 |
-
str: The content of the Wikipedia page if found and successfully fetched,
|
| 17 |
-
or an error message if no page is found or an exception occurs.
|
| 18 |
-
Raises:
|
| 19 |
-
Exception: If an error occurs while fetching the Wikipedia page.
|
| 20 |
-
"""
|
| 21 |
-
|
| 22 |
-
try:
|
| 23 |
-
titles = wiki.search(query)
|
| 24 |
-
if not titles or len(titles) == 0:
|
| 25 |
-
return "No Wikipedia page found for the given query."
|
| 26 |
-
|
| 27 |
-
page = wiki.page(titles[0])
|
| 28 |
-
url = page.url
|
| 29 |
-
response = requests.get(url)
|
| 30 |
-
response.raise_for_status()
|
| 31 |
-
return response.text
|
| 32 |
-
except Exception as e:
|
| 33 |
-
raise Exception(f"Error fetching Wikipedia page: {e}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tools/search_vector_store.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import wikipedia as wiki
|
| 2 |
+
from bs4 import BeautifulSoup
|
| 3 |
+
from langchain_core.tools import tool
|
| 4 |
+
from langchain_core.vectorstores import InMemoryVectorStore
|
| 5 |
+
from langchain_text_splitters import HTMLHeaderTextSplitter
|
| 6 |
+
from langchain_together import TogetherEmbeddings
|
| 7 |
+
|
| 8 |
+
vector_store = InMemoryVectorStore(TogetherEmbeddings())
|
| 9 |
+
retriever = vector_store.as_retriever()
|
| 10 |
+
text_splitter = HTMLHeaderTextSplitter(
|
| 11 |
+
headers_to_split_on=[("h2", "Section"), ("h3", "Subsection")],
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@tool(description="Search the vector store for data matching the given query.")
|
| 16 |
+
def search_vector_store(query: str) -> str:
|
| 17 |
+
"""
|
| 18 |
+
Searches the vector store for data matching the given query.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
query (str): The search query to look up on Wikipedia.
|
| 22 |
+
|
| 23 |
+
Returns:
|
| 24 |
+
str: The concatenated content of the retrieved documents or an error message if no
|
| 25 |
+
relevant page is found.
|
| 26 |
+
|
| 27 |
+
Raises:
|
| 28 |
+
Exception: Raised if there is any error while fetching data
|
| 29 |
+
"""
|
| 30 |
+
try:
|
| 31 |
+
print(f"Searching Wikipedia for: {query}")
|
| 32 |
+
titles = wiki.search(query)
|
| 33 |
+
if not titles or len(titles) == 0:
|
| 34 |
+
return "No Wikipedia page found for the given query."
|
| 35 |
+
print(f"Found titles: {titles}")
|
| 36 |
+
|
| 37 |
+
# Fetch the Wikipedia page
|
| 38 |
+
print(f"Fetching Wikipedia page for title: {titles[0]}")
|
| 39 |
+
page = wiki.page(titles[0])
|
| 40 |
+
|
| 41 |
+
# Parse the HTML content
|
| 42 |
+
soup = BeautifulSoup(page.html(), "html.parser")
|
| 43 |
+
content_div = soup.find(id="mw-content-text")
|
| 44 |
+
|
| 45 |
+
section = page.section("kek")
|
| 46 |
+
|
| 47 |
+
print("Splitting content...")
|
| 48 |
+
docs = text_splitter.split_text(content_div.decode())
|
| 49 |
+
|
| 50 |
+
print("Adding documents to vector store...")
|
| 51 |
+
doc_ids = vector_store.add_documents(docs)
|
| 52 |
+
print(f"Added {len(doc_ids)} documents to the vector store.")
|
| 53 |
+
|
| 54 |
+
print("Retrieving documents from vector store...")
|
| 55 |
+
retriever.invoke(input=query)
|
| 56 |
+
print(f"Retrieved {len(docs)} documents from the vector store.")
|
| 57 |
+
result = "\n".join([doc.page_content for doc in docs])
|
| 58 |
+
print(f"Result: {result}")
|
| 59 |
+
return result
|
| 60 |
+
|
| 61 |
+
except Exception as e:
|
| 62 |
+
raise Exception(f"Error fetching data: {e}")
|
vector_store.py
DELETED
|
@@ -1,58 +0,0 @@
|
|
| 1 |
-
from langchain_core.documents import Document
|
| 2 |
-
from langchain_core.messages import SystemMessage
|
| 3 |
-
from langchain_core.pydantic_v1 import BaseModel, Field
|
| 4 |
-
from langchain_core.vectorstores.in_memory import InMemoryVectorStore
|
| 5 |
-
from langchain_together import ChatTogether
|
| 6 |
-
from langchain_together.embeddings import TogetherEmbeddings
|
| 7 |
-
|
| 8 |
-
from questions import get_questions
|
| 9 |
-
from tools import get_wikipedia_page
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
class ResponseSchema(BaseModel):
|
| 13 |
-
search_query: str | None = Field(
|
| 14 |
-
...,
|
| 15 |
-
description="The search query to be used for Wikipedia search, or None if no search is needed.",
|
| 16 |
-
)
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
if __name__ == "__main__":
|
| 20 |
-
vector_store = InMemoryVectorStore(TogetherEmbeddings())
|
| 21 |
-
model = model = ChatTogether(
|
| 22 |
-
model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
|
| 23 |
-
temperature=0,
|
| 24 |
-
max_retries=3,
|
| 25 |
-
timeout=15,
|
| 26 |
-
).with_structured_output(schema=ResponseSchema, method="json_schema", strict=True)
|
| 27 |
-
questions = get_questions(limit=3)
|
| 28 |
-
print(f"Fetched {len(questions)} questions.")
|
| 29 |
-
|
| 30 |
-
sys_msg = SystemMessage(
|
| 31 |
-
content=(
|
| 32 |
-
"You are an intelligent and resourceful assistant. "
|
| 33 |
-
"Your task is to determine if a Wikipedia search is necessary to provide an accurate and comprehensive answer to the given question. "
|
| 34 |
-
),
|
| 35 |
-
)
|
| 36 |
-
for question in questions:
|
| 37 |
-
print(f"Question: {question.question}")
|
| 38 |
-
response: ResponseSchema = model.invoke([sys_msg, question.question])
|
| 39 |
-
|
| 40 |
-
if response.search_query is None:
|
| 41 |
-
print("No search query needed.")
|
| 42 |
-
continue
|
| 43 |
-
|
| 44 |
-
print(f"Search query: {response.search_query}")
|
| 45 |
-
wikipedia_page = get_wikipedia_page(response.search_query)
|
| 46 |
-
doc = Document(page_content=wikipedia_page)
|
| 47 |
-
vector_store.add_documents([doc])
|
| 48 |
-
|
| 49 |
-
while True:
|
| 50 |
-
user_input = input("Enter your message (or 'exit' to quit): ")
|
| 51 |
-
if user_input.lower() in ["exit", "quit", "q"]:
|
| 52 |
-
break
|
| 53 |
-
|
| 54 |
-
# Search the vector store for relevant documents
|
| 55 |
-
docs = vector_store.similarity_search(user_input)
|
| 56 |
-
print(f"Found {len(docs)} documents in the vector store.")
|
| 57 |
-
for doc in docs:
|
| 58 |
-
print(f"Document: {doc.page_content}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|