import base64 import os from typing import cast import requests from langchain_core.messages import HumanMessage from langchain_core.tools import tool from langchain_openai import ChatOpenAI import questions @tool(parse_docstring=True) def handle_image(query: str, file_name: str) -> str: """ Handle the image query and return the response. Args: query (str): The question to be answered. file_name (str): The name of the image file. Returns: str: The response from the LLM. """ vision_llm = ChatOpenAI( model="gpt-4o", temperature=0.0, max_retries=3, ) try: image_base64 = base64.encodebytes(questions.get_file(file_name.split(".")[0])) output = vision_llm.invoke( [ HumanMessage( content=[ {"type": "text", "text": query}, { "type": "image_url", "image_url": { "url": f"data:image/png;base64,{image_base64.decode('utf-8')}", }, }, ] ) ], ) return cast(str, output.content) except Exception as e: return "Error processing image: " + str(e) if __name__ == "__main__": def get_file(task_id: str) -> bytes: API_URL = os.getenv("API_URL") or "https://agents-course-unit4-scoring.hf.space" url = f"{API_URL}/files/{task_id}" print(f"Fetching file from: {url}") response = requests.get(url, timeout=15) response.raise_for_status() return response.content llm = ChatOpenAI( model="gpt-4o", temperature=0.0, max_retries=3, ) question = { "task_id": "cca530fc-4052-43b2-b130-b30968d8aa44", "question": "Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.", "file_name": "cca530fc-4052-43b2-b130-b30968d8aa44.png", } image_base64 = base64.encodebytes(get_file(question["task_id"])) try: output = llm.invoke( [ HumanMessage( content=[ {"type": "text", "text": question["question"]}, { "type": "image_url", "image_url": { "url": f"data:image/png;base64,{image_base64.decode('utf-8')}", }, }, ] ) ], ) print(f"Response: {output.content}") except Exception as e: print("Agent error: ", e)