Andrei Sferdian
Add vision capabilities
b49d1d5
Raw History Blame Contribute Delete
7.56 kB
import mimetypes
import os
from typing import Annotated, TypedDict
import requests
from langchain_community.tools.google_serper import GoogleSerperResults
from langchain_core.messages import AnyMessage, HumanMessage
from langchain_openai import ChatOpenAI
from langchain_together import ChatTogether
from langgraph.graph import START, StateGraph
from langgraph.graph.message import add_messages
from langgraph.prebuilt import create_react_agent
import questions
from tools import (
GetYouTubeTranscript,
add,
categorize_foods,
check_commutativity,
divide,
execute_python_code,
find_wiki_page,
floor_divide,
get_text_file,
get_wiki_page_section_by_title,
handle_image,
load_excel_file,
modulus,
multiply,
power,
subtract,
sum_rows_cols,
)
tools = [
GetYouTubeTranscript(),
GoogleSerperResults(),
execute_python_code,
find_wiki_page,
get_wiki_page_section_by_title,
load_excel_file,
get_text_file,
add,
subtract,
multiply,
divide,
power,
modulus,
floor_divide,
sum_rows_cols,
check_commutativity,
categorize_foods,
handle_image,
]
mode = os.getenv("MODE", "free").lower()
model = (
ChatOpenAI(
model="gpt-4.1",
temperature=0.0,
max_retries=3,
)
if mode == "paid"
else ChatTogether(
model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
temperature=0.0,
max_retries=3,
)
).bind_tools(tools)
prompt = (
"You are an agent that will undergo the GAIA test.\n"
"You will be given a question and you will have to answer it using the tools provided.\n"
"Respond in the requested format, if specified, otherwise use the short answer style. I.E. Question: What is the capital of France? Answer: Paris.\n"
"When searching for information, always follow this specific sequence:\n"
" a. Use the 'google_serper_results_json' tool to search for relevant information.\n"
" Provide the query as input to the tool.\n"
" The tool will return a JSON string with the search results.\n"
" b. If the required information is not found in the search results, use the find_wiki_page tool to locate the relevant Wikipedia page.\n"
" The response will be a JSON string with the page's summary and sections.\n"
" c. If the Wikipedia page exists, and the summary does not contain the required info, use get_wiki_page_section_by_title tool to get the content of the desired section.\n"
"If the user asks for an Excel file, use the load_excel_file tool.\n"
"If you need to sum rows and columns in an Excel file, use the sum_rows_cols tool.\n"
"If the user provides a Python code snippet, follow this sequence:\n"
" a. Use the 'get_text_file' tool to download the code file if a file name or URL is provided.\n"
" b. Use the 'execute_python_code' tool to run the Python code.\n"
" Provide the code as input to the tool.\n"
"For reasoning tasks, adopt a step-by-step approach, but only output the final result.\n"
"Do not explain your reasoning, just provide the answer!\n"
"Do not generate any code, only use the tools provided!\n"
)
class GraphState(TypedDict):
messages: Annotated[list[AnyMessage], add_messages]
is_reverse: bool
file_name: str | None
question: questions.Question
def validate_state(state: GraphState):
if not state.get("question") or not state["question"].get("question"):
raise ValueError("Question is not provided or is empty.")
file_name = state["question"].get("file_name")
if file_name and not _is_supported_file_type(file_name):
raise ValueError(f"File name '{file_name}' is not a supported file type.")
return {
"messages": [HumanMessage(content=state["question"]["question"])],
"file_name": state["question"].get("file_name"),
}
def _is_supported_file_type(file_name: str) -> bool:
response = mimetypes.guess_type(file_name)
if (
not response
or not response[0]
or any(response[0].startswith(prefix) for prefix in ["audio", "video"])
):
return False
return True
log_level = os.getenv("LOG_LEVEL", "info").lower()
react_agent = create_react_agent(
model=model,
tools=tools,
prompt=prompt,
debug=log_level == "debug",
)
def detect_reverse(state: GraphState):
question = state["messages"][-1]
model_ = ChatTogether(
model="meta-llama/Llama-3.3-70B-Instruct-Turbo-Free",
temperature=0,
max_tokens=2000,
max_retries=3,
)
prompt = f"""Detect if the following text is written backwards.
If it is, respond with "yes", otherwise respond with "no".
Question: {question.content}"""
response = model_.invoke(
[
HumanMessage(content=prompt),
]
)
return {
"messages": [question],
"is_reverse": response.content.strip().lower() == "yes",
}
def prepare(state: GraphState):
question = state["messages"][-1].content
print(f"Preparing question: {question}")
if state.get("is_reverse", False):
print("Question is in reverse order, reversing it...")
question = question[::-1]
file_name = state.get("file_name")
if file_name:
return {
"messages": [
HumanMessage(content=f"Question: {question}\nFile name: {file_name}")
],
}
return {
"messages": [HumanMessage(content=question)],
}
graph = StateGraph(GraphState)
graph.add_node("validate_state", validate_state)
graph.add_node(
"detect_reverse",
detect_reverse,
)
graph.add_node(
"prepare",
prepare,
)
graph.add_node("react", react_agent)
graph.add_edge(START, "validate_state")
graph.add_edge("validate_state", "detect_reverse")
graph.add_edge("detect_reverse", "prepare")
graph.add_edge("prepare", "react")
agent = graph.compile().with_retry(
retry_if_exception_type=(requests.HTTPError,),
wait_exponential_jitter=True,
stop_after_attempt=3,
exponential_jitter_params={
"initial": 5.0,
"max": 60.0,
},
)
if __name__ == "__main__":
question = {
"task_id": "3cef3a44-215e-4aed-8e3b-b1e3f08063b7",
"question": "I'm making a grocery list for my mom, but she's a professor of botany and she's a real stickler when it comes to categorizing things. I need to add different foods to different categories on the grocery list, but if I make a mistake, she won't buy anything inserted in the wrong category. Here's the list I have so far:\n\nmilk, eggs, flour, whole bean coffee, Oreos, sweet potatoes, fresh basil, plums, green beans, rice, corn, bell pepper, whole allspice, acorns, broccoli, celery, zucchini, lettuce, peanuts\n\nI need to make headings for the fruits and vegetables. Could you please create a list of just the vegetables from my list? If you could do that, then I can figure out how to categorize the rest of the list into the appropriate categories. But remember that my mom is a real stickler, so make sure that no botanical fruits end up on the vegetable list, or she won't get them when she's at the store. Please alphabetize the list of vegetables, and place each item in a comma separated list.",
"file_name": "",
}
try:
result = agent.invoke(
{
"question": question,
}
)
print("Result:", result["messages"][-1].content)
except Exception as e:
print(f"Error: {e}")