Andrei Sferdian commited on
Commit
993b644
·
1 Parent(s): 6614d6e

Experiment with vector store

Browse files
Files changed (5) hide show
  1. agent.py +2 -25
  2. questions.py +45 -0
  3. requirements.txt +1 -0
  4. tools/get_wikipedia_page.py +3 -1
  5. vector_store.py +58 -0
agent.py CHANGED
@@ -1,5 +1,4 @@
1
  import asyncio
2
- import random
3
  from functools import reduce
4
 
5
  import filetype
@@ -11,6 +10,7 @@ from langgraph.graph import StateGraph
11
  from langgraph.prebuilt import create_react_agent
12
  from langgraph.prebuilt.chat_agent_executor import AgentState
13
 
 
14
  from tools import GetYouTubeTranscript, get_wikipedia_page
15
 
16
  tools = [
@@ -47,29 +47,6 @@ class Urls:
47
  return f"{cls.BASE_URL}/files/{task_id}"
48
 
49
 
50
- def get_questions(limit: int = None) -> list[dict]:
51
- try:
52
- response = requests.get(Urls.questions(), timeout=15)
53
- response.raise_for_status()
54
- questions_data = response.json()
55
- if not questions_data:
56
- return []
57
-
58
- return random.sample(
59
- questions_data,
60
- min(
61
- limit,
62
- len(questions_data),
63
- )
64
- if limit is not None
65
- else len(questions_data),
66
- )
67
-
68
- except Exception as e:
69
- print(f"Error fetching questions: {e}")
70
- raise
71
-
72
-
73
  if __name__ == "__main__":
74
  # while True:
75
  # Get user input
@@ -77,7 +54,7 @@ if __name__ == "__main__":
77
  # if user_input.lower() in ["exit", "quit", "q"]:
78
  # break
79
 
80
- questions = get_questions()
81
  print(f"Fetched {len(questions)} questions.")
82
  # print(f"Questions: {questions}")
83
  question = next((q for q in questions if q.get("file_name")), None)
 
1
  import asyncio
 
2
  from functools import reduce
3
 
4
  import filetype
 
10
  from langgraph.prebuilt import create_react_agent
11
  from langgraph.prebuilt.chat_agent_executor import AgentState
12
 
13
+ import questions
14
  from tools import GetYouTubeTranscript, get_wikipedia_page
15
 
16
  tools = [
 
47
  return f"{cls.BASE_URL}/files/{task_id}"
48
 
49
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  if __name__ == "__main__":
51
  # while True:
52
  # Get user input
 
54
  # if user_input.lower() in ["exit", "quit", "q"]:
55
  # break
56
 
57
+ questions = questions.get_questions()
58
  print(f"Fetched {len(questions)} questions.")
59
  # print(f"Questions: {questions}")
60
  question = next((q for q in questions if q.get("file_name")), None)
questions.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+
4
+ import requests
5
+ from pydantic import BaseModel, Field
6
+
7
+
8
+ class Question(BaseModel):
9
+ task_id: str = Field(..., min_length=1)
10
+ question: str = Field(..., min_length=1)
11
+ file_name: str = Field(default="")
12
+
13
+
14
+ def get_questions(limit: int = None) -> list[Question]:
15
+ base_url = os.getenv("BASE_URL")
16
+ if base_url is None:
17
+ raise ValueError("BASE_URL environment variable is not set")
18
+
19
+ try:
20
+ response = requests.get(base_url + "/questions", timeout=15)
21
+ response.raise_for_status()
22
+ questions_data = [
23
+ Question(
24
+ task_id=q["task_id"],
25
+ question=q["question"],
26
+ file_name=q["file_name"],
27
+ )
28
+ for q in response.json()
29
+ ]
30
+ if not questions_data:
31
+ return []
32
+
33
+ return random.sample(
34
+ questions_data,
35
+ min(
36
+ limit,
37
+ len(questions_data),
38
+ )
39
+ if limit is not None
40
+ else len(questions_data),
41
+ )
42
+
43
+ except Exception as e:
44
+ print(f"Error fetching questions: {e}")
45
+ raise
requirements.txt CHANGED
@@ -29,6 +29,7 @@ emoji==2.14.1
29
  et_xmlfile==2.0.0
30
  eval_type_backport==0.2.2
31
  exceptiongroup==1.2.2
 
32
  fastapi==0.115.12
33
  ffmpy==0.5.0
34
  filelock==3.18.0
 
29
  et_xmlfile==2.0.0
30
  eval_type_backport==0.2.2
31
  exceptiongroup==1.2.2
32
+ faiss-cpu==1.11.0
33
  fastapi==0.115.12
34
  ffmpy==0.5.0
35
  filelock==3.18.0
tools/get_wikipedia_page.py CHANGED
@@ -15,6 +15,8 @@ def get_wikipedia_page(query: str) -> str:
15
  Returns:
16
  str: The content of the Wikipedia page if found and successfully fetched,
17
  or an error message if no page is found or an exception occurs.
 
 
18
  """
19
 
20
  try:
@@ -28,4 +30,4 @@ def get_wikipedia_page(query: str) -> str:
28
  response.raise_for_status()
29
  return response.text
30
  except Exception as e:
31
- return f"Error fetching Wikipedia page: {e}"
 
15
  Returns:
16
  str: The content of the Wikipedia page if found and successfully fetched,
17
  or an error message if no page is found or an exception occurs.
18
+ Raises:
19
+ Exception: If an error occurs while fetching the Wikipedia page.
20
  """
21
 
22
  try:
 
30
  response.raise_for_status()
31
  return response.text
32
  except Exception as e:
33
+ raise Exception(f"Error fetching Wikipedia page: {e}")
vector_store.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain_core.documents import Document
2
+ from langchain_core.messages import SystemMessage
3
+ from langchain_core.pydantic_v1 import BaseModel, Field
4
+ from langchain_core.vectorstores.in_memory import InMemoryVectorStore
5
+ from langchain_together import ChatTogether
6
+ from langchain_together.embeddings import TogetherEmbeddings
7
+
8
+ from questions import get_questions
9
+ from tools import get_wikipedia_page
10
+
11
+
12
+ class ResponseSchema(BaseModel):
13
+ search_query: str | None = Field(
14
+ ...,
15
+ description="The search query to be used for Wikipedia search, or None if no search is needed.",
16
+ )
17
+
18
+
19
+ if __name__ == "__main__":
20
+ vector_store = InMemoryVectorStore(TogetherEmbeddings())
21
+ model = model = ChatTogether(
22
+ model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
23
+ temperature=0,
24
+ max_retries=3,
25
+ timeout=15,
26
+ ).with_structured_output(schema=ResponseSchema, method="json_schema", strict=True)
27
+ questions = get_questions(limit=3)
28
+ print(f"Fetched {len(questions)} questions.")
29
+
30
+ sys_msg = SystemMessage(
31
+ content=(
32
+ "You are an intelligent and resourceful assistant. "
33
+ "Your task is to determine if a Wikipedia search is necessary to provide an accurate and comprehensive answer to the given question. "
34
+ ),
35
+ )
36
+ for question in questions:
37
+ print(f"Question: {question.question}")
38
+ response: ResponseSchema = model.invoke([sys_msg, question.question])
39
+
40
+ if response.search_query is None:
41
+ print("No search query needed.")
42
+ continue
43
+
44
+ print(f"Search query: {response.search_query}")
45
+ wikipedia_page = get_wikipedia_page(response.search_query)
46
+ doc = Document(page_content=wikipedia_page)
47
+ vector_store.add_documents([doc])
48
+
49
+ while True:
50
+ user_input = input("Enter your message (or 'exit' to quit): ")
51
+ if user_input.lower() in ["exit", "quit", "q"]:
52
+ break
53
+
54
+ # Search the vector store for relevant documents
55
+ docs = vector_store.similarity_search(user_input)
56
+ print(f"Found {len(docs)} documents in the vector store.")
57
+ for doc in docs:
58
+ print(f"Document: {doc.page_content}")