Andrei Sferdian commited on
Commit ·
993b644
1
Parent(s): 6614d6e
Experiment with vector store
Browse files- agent.py +2 -25
- questions.py +45 -0
- requirements.txt +1 -0
- tools/get_wikipedia_page.py +3 -1
- vector_store.py +58 -0
agent.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
| 1 |
import asyncio
|
| 2 |
-
import random
|
| 3 |
from functools import reduce
|
| 4 |
|
| 5 |
import filetype
|
|
@@ -11,6 +10,7 @@ from langgraph.graph import StateGraph
|
|
| 11 |
from langgraph.prebuilt import create_react_agent
|
| 12 |
from langgraph.prebuilt.chat_agent_executor import AgentState
|
| 13 |
|
|
|
|
| 14 |
from tools import GetYouTubeTranscript, get_wikipedia_page
|
| 15 |
|
| 16 |
tools = [
|
|
@@ -47,29 +47,6 @@ class Urls:
|
|
| 47 |
return f"{cls.BASE_URL}/files/{task_id}"
|
| 48 |
|
| 49 |
|
| 50 |
-
def get_questions(limit: int = None) -> list[dict]:
|
| 51 |
-
try:
|
| 52 |
-
response = requests.get(Urls.questions(), timeout=15)
|
| 53 |
-
response.raise_for_status()
|
| 54 |
-
questions_data = response.json()
|
| 55 |
-
if not questions_data:
|
| 56 |
-
return []
|
| 57 |
-
|
| 58 |
-
return random.sample(
|
| 59 |
-
questions_data,
|
| 60 |
-
min(
|
| 61 |
-
limit,
|
| 62 |
-
len(questions_data),
|
| 63 |
-
)
|
| 64 |
-
if limit is not None
|
| 65 |
-
else len(questions_data),
|
| 66 |
-
)
|
| 67 |
-
|
| 68 |
-
except Exception as e:
|
| 69 |
-
print(f"Error fetching questions: {e}")
|
| 70 |
-
raise
|
| 71 |
-
|
| 72 |
-
|
| 73 |
if __name__ == "__main__":
|
| 74 |
# while True:
|
| 75 |
# Get user input
|
|
@@ -77,7 +54,7 @@ if __name__ == "__main__":
|
|
| 77 |
# if user_input.lower() in ["exit", "quit", "q"]:
|
| 78 |
# break
|
| 79 |
|
| 80 |
-
questions = get_questions()
|
| 81 |
print(f"Fetched {len(questions)} questions.")
|
| 82 |
# print(f"Questions: {questions}")
|
| 83 |
question = next((q for q in questions if q.get("file_name")), None)
|
|
|
|
| 1 |
import asyncio
|
|
|
|
| 2 |
from functools import reduce
|
| 3 |
|
| 4 |
import filetype
|
|
|
|
| 10 |
from langgraph.prebuilt import create_react_agent
|
| 11 |
from langgraph.prebuilt.chat_agent_executor import AgentState
|
| 12 |
|
| 13 |
+
import questions
|
| 14 |
from tools import GetYouTubeTranscript, get_wikipedia_page
|
| 15 |
|
| 16 |
tools = [
|
|
|
|
| 47 |
return f"{cls.BASE_URL}/files/{task_id}"
|
| 48 |
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
if __name__ == "__main__":
|
| 51 |
# while True:
|
| 52 |
# Get user input
|
|
|
|
| 54 |
# if user_input.lower() in ["exit", "quit", "q"]:
|
| 55 |
# break
|
| 56 |
|
| 57 |
+
questions = questions.get_questions()
|
| 58 |
print(f"Fetched {len(questions)} questions.")
|
| 59 |
# print(f"Questions: {questions}")
|
| 60 |
question = next((q for q in questions if q.get("file_name")), None)
|
questions.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import random
|
| 3 |
+
|
| 4 |
+
import requests
|
| 5 |
+
from pydantic import BaseModel, Field
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class Question(BaseModel):
|
| 9 |
+
task_id: str = Field(..., min_length=1)
|
| 10 |
+
question: str = Field(..., min_length=1)
|
| 11 |
+
file_name: str = Field(default="")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def get_questions(limit: int = None) -> list[Question]:
|
| 15 |
+
base_url = os.getenv("BASE_URL")
|
| 16 |
+
if base_url is None:
|
| 17 |
+
raise ValueError("BASE_URL environment variable is not set")
|
| 18 |
+
|
| 19 |
+
try:
|
| 20 |
+
response = requests.get(base_url + "/questions", timeout=15)
|
| 21 |
+
response.raise_for_status()
|
| 22 |
+
questions_data = [
|
| 23 |
+
Question(
|
| 24 |
+
task_id=q["task_id"],
|
| 25 |
+
question=q["question"],
|
| 26 |
+
file_name=q["file_name"],
|
| 27 |
+
)
|
| 28 |
+
for q in response.json()
|
| 29 |
+
]
|
| 30 |
+
if not questions_data:
|
| 31 |
+
return []
|
| 32 |
+
|
| 33 |
+
return random.sample(
|
| 34 |
+
questions_data,
|
| 35 |
+
min(
|
| 36 |
+
limit,
|
| 37 |
+
len(questions_data),
|
| 38 |
+
)
|
| 39 |
+
if limit is not None
|
| 40 |
+
else len(questions_data),
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
except Exception as e:
|
| 44 |
+
print(f"Error fetching questions: {e}")
|
| 45 |
+
raise
|
requirements.txt
CHANGED
|
@@ -29,6 +29,7 @@ emoji==2.14.1
|
|
| 29 |
et_xmlfile==2.0.0
|
| 30 |
eval_type_backport==0.2.2
|
| 31 |
exceptiongroup==1.2.2
|
|
|
|
| 32 |
fastapi==0.115.12
|
| 33 |
ffmpy==0.5.0
|
| 34 |
filelock==3.18.0
|
|
|
|
| 29 |
et_xmlfile==2.0.0
|
| 30 |
eval_type_backport==0.2.2
|
| 31 |
exceptiongroup==1.2.2
|
| 32 |
+
faiss-cpu==1.11.0
|
| 33 |
fastapi==0.115.12
|
| 34 |
ffmpy==0.5.0
|
| 35 |
filelock==3.18.0
|
tools/get_wikipedia_page.py
CHANGED
|
@@ -15,6 +15,8 @@ def get_wikipedia_page(query: str) -> str:
|
|
| 15 |
Returns:
|
| 16 |
str: The content of the Wikipedia page if found and successfully fetched,
|
| 17 |
or an error message if no page is found or an exception occurs.
|
|
|
|
|
|
|
| 18 |
"""
|
| 19 |
|
| 20 |
try:
|
|
@@ -28,4 +30,4 @@ def get_wikipedia_page(query: str) -> str:
|
|
| 28 |
response.raise_for_status()
|
| 29 |
return response.text
|
| 30 |
except Exception as e:
|
| 31 |
-
|
|
|
|
| 15 |
Returns:
|
| 16 |
str: The content of the Wikipedia page if found and successfully fetched,
|
| 17 |
or an error message if no page is found or an exception occurs.
|
| 18 |
+
Raises:
|
| 19 |
+
Exception: If an error occurs while fetching the Wikipedia page.
|
| 20 |
"""
|
| 21 |
|
| 22 |
try:
|
|
|
|
| 30 |
response.raise_for_status()
|
| 31 |
return response.text
|
| 32 |
except Exception as e:
|
| 33 |
+
raise Exception(f"Error fetching Wikipedia page: {e}")
|
vector_store.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langchain_core.documents import Document
|
| 2 |
+
from langchain_core.messages import SystemMessage
|
| 3 |
+
from langchain_core.pydantic_v1 import BaseModel, Field
|
| 4 |
+
from langchain_core.vectorstores.in_memory import InMemoryVectorStore
|
| 5 |
+
from langchain_together import ChatTogether
|
| 6 |
+
from langchain_together.embeddings import TogetherEmbeddings
|
| 7 |
+
|
| 8 |
+
from questions import get_questions
|
| 9 |
+
from tools import get_wikipedia_page
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ResponseSchema(BaseModel):
|
| 13 |
+
search_query: str | None = Field(
|
| 14 |
+
...,
|
| 15 |
+
description="The search query to be used for Wikipedia search, or None if no search is needed.",
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
if __name__ == "__main__":
|
| 20 |
+
vector_store = InMemoryVectorStore(TogetherEmbeddings())
|
| 21 |
+
model = model = ChatTogether(
|
| 22 |
+
model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
|
| 23 |
+
temperature=0,
|
| 24 |
+
max_retries=3,
|
| 25 |
+
timeout=15,
|
| 26 |
+
).with_structured_output(schema=ResponseSchema, method="json_schema", strict=True)
|
| 27 |
+
questions = get_questions(limit=3)
|
| 28 |
+
print(f"Fetched {len(questions)} questions.")
|
| 29 |
+
|
| 30 |
+
sys_msg = SystemMessage(
|
| 31 |
+
content=(
|
| 32 |
+
"You are an intelligent and resourceful assistant. "
|
| 33 |
+
"Your task is to determine if a Wikipedia search is necessary to provide an accurate and comprehensive answer to the given question. "
|
| 34 |
+
),
|
| 35 |
+
)
|
| 36 |
+
for question in questions:
|
| 37 |
+
print(f"Question: {question.question}")
|
| 38 |
+
response: ResponseSchema = model.invoke([sys_msg, question.question])
|
| 39 |
+
|
| 40 |
+
if response.search_query is None:
|
| 41 |
+
print("No search query needed.")
|
| 42 |
+
continue
|
| 43 |
+
|
| 44 |
+
print(f"Search query: {response.search_query}")
|
| 45 |
+
wikipedia_page = get_wikipedia_page(response.search_query)
|
| 46 |
+
doc = Document(page_content=wikipedia_page)
|
| 47 |
+
vector_store.add_documents([doc])
|
| 48 |
+
|
| 49 |
+
while True:
|
| 50 |
+
user_input = input("Enter your message (or 'exit' to quit): ")
|
| 51 |
+
if user_input.lower() in ["exit", "quit", "q"]:
|
| 52 |
+
break
|
| 53 |
+
|
| 54 |
+
# Search the vector store for relevant documents
|
| 55 |
+
docs = vector_store.similarity_search(user_input)
|
| 56 |
+
print(f"Found {len(docs)} documents in the vector store.")
|
| 57 |
+
for doc in docs:
|
| 58 |
+
print(f"Document: {doc.page_content}")
|