File size: 2,877 Bytes
b49d1d5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 | import base64
import os
from typing import cast
import requests
from langchain_core.messages import HumanMessage
from langchain_core.tools import tool
from langchain_openai import ChatOpenAI
import questions
@tool(parse_docstring=True)
def handle_image(query: str, file_name: str) -> str:
"""
Handle the image query and return the response.
Args:
query (str): The question to be answered.
file_name (str): The name of the image file.
Returns:
str: The response from the LLM.
"""
vision_llm = ChatOpenAI(
model="gpt-4o",
temperature=0.0,
max_retries=3,
)
try:
image_base64 = base64.encodebytes(questions.get_file(file_name.split(".")[0]))
output = vision_llm.invoke(
[
HumanMessage(
content=[
{"type": "text", "text": query},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_base64.decode('utf-8')}",
},
},
]
)
],
)
return cast(str, output.content)
except Exception as e:
return "Error processing image: " + str(e)
if __name__ == "__main__":
def get_file(task_id: str) -> bytes:
API_URL = os.getenv("API_URL") or "https://agents-course-unit4-scoring.hf.space"
url = f"{API_URL}/files/{task_id}"
print(f"Fetching file from: {url}")
response = requests.get(url, timeout=15)
response.raise_for_status()
return response.content
llm = ChatOpenAI(
model="gpt-4o",
temperature=0.0,
max_retries=3,
)
question = {
"task_id": "cca530fc-4052-43b2-b130-b30968d8aa44",
"question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.",
"file_name": "cca530fc-4052-43b2-b130-b30968d8aa44.png",
}
image_base64 = base64.encodebytes(get_file(question["task_id"]))
try:
output = llm.invoke(
[
HumanMessage(
content=[
{"type": "text", "text": question["question"]},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{image_base64.decode('utf-8')}",
},
},
]
)
],
)
print(f"Response: {output.content}")
except Exception as e:
print("Agent error: ", e)
|