Andrei Sferdian
Add vision capabilities
b49d1d5
Raw History Blame Contribute Delete
2.88 kB
import base64
import os
from typing import cast
import requests
from langchain_core.messages import HumanMessage
from langchain_core.tools import tool
from langchain_openai import ChatOpenAI
import questions
@tool(parse_docstring=True)
def handle_image(query: str, file_name: str) -> str:
"""
Handle the image query and return the response.
Args:
query (str): The question to be answered.
file_name (str): The name of the image file.
Returns:
str: The response from the LLM.
"""
vision_llm = ChatOpenAI(
model="gpt-4o",
temperature=0.0,
max_retries=3,
)
try:
image_base64 = base64.encodebytes(questions.get_file(file_name.split(".")[0]))
output = vision_llm.invoke(
[
HumanMessage(
content=[
{"type": "text", "text": query},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_base64.decode('utf-8')}",
},
},
]
)
],
)
return cast(str, output.content)
except Exception as e:
return "Error processing image: " + str(e)
if __name__ == "__main__":
def get_file(task_id: str) -> bytes:
API_URL = os.getenv("API_URL") or "https://agents-course-unit4-scoring.hf.space"
url = f"{API_URL}/files/{task_id}"
print(f"Fetching file from: {url}")
response = requests.get(url, timeout=15)
response.raise_for_status()
return response.content
llm = ChatOpenAI(
model="gpt-4o",
temperature=0.0,
max_retries=3,
)
question = {
"task_id": "cca530fc-4052-43b2-b130-b30968d8aa44",
"question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
"file_name": "cca530fc-4052-43b2-b130-b30968d8aa44.png",
}
image_base64 = base64.encodebytes(get_file(question["task_id"]))
try:
output = llm.invoke(
[
HumanMessage(
content=[
{"type": "text", "text": question["question"]},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_base64.decode('utf-8')}",
},
},
]
)
],
)
print(f"Response: {output.content}")
except Exception as e:
print("Agent error: ", e)