import os from typing import List, Dict, Any, Optional from dotenv import load_dotenv from langgraph.graph import START, StateGraph, MessagesState from langgraph.prebuilt import tools_condition, ToolNode from langchain_core.tools import tool from langchain_core.messages import SystemMessage, HumanMessage from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings from langchain_google_genai import ChatGoogleGenerativeAI from langchain_community.document_loaders import WikipediaLoader, ArxivLoader, YoutubeLoader from langchain_community.document_loaders.youtube import TranscriptFormat from langchain_community.tools.tavily_search import TavilySearchResults from langchain_community.vectorstores import SupabaseVectorStore from langchain.tools.retriever import create_retriever_tool from supabase.client import Client, create_client import tempfile import re import json import requests from urllib.parse import urlparse import pytesseract from PIL import Image, ImageDraw, ImageFont, ImageEnhance, ImageFilter import cmath import pandas as pd import uuid import numpy as np import base64 import io import contextlib import torch load_dotenv() @tool def add(a: int, b: int) -> int: """ Add two numbers Args: a: First int b: Second int """ return a + b @tool def subtract(a: int, b: int) -> int: """ Subtract two numbers Args: a: First int b: Second int """ return a - b @tool def multiply(a: int, b: int) -> int: """ Multply two numbers Args: a: First int b: Second int """ return a * b @tool def divide(a: int, b: int) -> int: """ Divide two numbers (Rounded into int) Args: a: First int b: Second int """ return int(a / b) @tool def modulus(a: int, b: int) -> int: """ Get the modulus of two numbers Args: a: First int b: Second int """ return a % b @tool def wiki_search(query: str) -> dict: """ Search wikipedia for a query Args: query: Text """ search_docs = WikipediaLoader(query = query, load_max_docs = 3).load() formatted_search_docs = "\n\n---\n\n".join( [ f'\n{doc.page_content}\n' for doc in search_docs ] ) return {"wiki_results" : formatted_search_docs} @tool def web_search(query: str) -> dict: """ Perform web search to get content from external sources Args: query: Text """ search_docs = TavilySearchResults(max_result = 3).invoke({'query' : query}) formatted_search_docs = "\n\n---\n\n".join( [ f'\n{doc.get("content", "")}\n' for doc in search_docs ] ) return {"web_results" : formatted_search_docs} @tool def fetch_arxiv_paper(paper_id_or_url: str) -> dict: """ Fetches a specific Arxiv paper when given its unique ID (e.g., '2001.08361') or its full URL. Use this tool when the user provides a direct link or paper ID. Args: paper_id_or_url: The Arxiv ID or the full URL of the paper. """ try: # Use regex to find the Arxiv ID from a URL # This looks for patterns like '2001.08361' or '1706.03762' match = re.search(r'(\d{4}\.\d{4,5})', paper_id_or_url) if match: paper_id = match.group(1) else: # Assume the input is already just the ID paper_id = paper_id_or_url # Use the extracted ID to load the specific document loader = ArxivLoader(query=paper_id, load_max_docs=1) docs = loader.load() if not docs: return {"error": f"Could not find a paper with ID: {paper_id}"} # Format the output using the structured method doc = docs[0] title = doc.metadata.get('Title', 'No Title Found') authors = doc.metadata.get('Authors', 'No Authors Found') summary = doc.metadata.get('Summary', 'No Summary Found').replace('\n', ' ') doc_string = ( f"\n" f" {title}\n" f" {authors}\n" f" {summary}\n" f"" ) return {"arxiv_paper": doc_string} except Exception as e: return {"error": f"Failed to fetch Arxiv paper: {str(e)}"} @tool def arxiv_search(query: str) -> dict: """ Search Arxiv for a query with a maximum of 3 results. Args: query: Text to search for on Arxiv. """ try: # Load the documents as before search_docs = ArxivLoader(query=query, load_max_docs=3).load() # Format the output using the CORRECT metadata keys provided by ArxivLoader formatted_docs = [] for doc in search_docs: # Safely get each piece of metadata using .get() title = doc.metadata.get('Title', 'No Title Found') authors = doc.metadata.get('Authors', 'No Authors Found') published_date = doc.metadata.get('Published', 'No Date Found') summary = doc.metadata.get('Summary', 'No Summary Found').replace('\n', ' ') # Remove newlines for cleaner output # Create a structured string that is easy for the LLM to parse doc_string = ( f"\n" f" {title}\n" f" {authors}\n" f" {published_date}\n" f" {summary}\n" f"" ) formatted_docs.append(doc_string) return {"arxiv_results": "\n\n---\n\n".join(formatted_docs)} except Exception as e: # Return a clear error message if something goes wrong return {"error": f"Arxiv search failed: {str(e)}"} @tool def read_file(file_path: str) -> dict: """ Reads the content from a local file path provided with the user's question. Supports file types: .txt, .csv, .xlsx (Excel), .py (Python), and image files (.png, .jpg, .jpeg). For images, it returns a Base64 encoded string for the LLM to analyze. Args: file_path: The local path to the file (e.g., 'validation/image.png'). """ try: if file_path.lower().endswith('.xlsx'): df = pd.read_excel(file_path) return {"file_content": df.to_string()} elif file_path.lower().endswith('.csv'): df = pd.read_csv(file_path) return {"file_content": df.to_string()} elif file_path.lower().endswith(('.txt', '.py')): with open(file_path, 'r', encoding='utf-8') as f: return {"file_content": f.read()} elif file_path.lower().endswith(('.png', '.jpg', '.jpeg')): with open(file_path, "rb") as image_file: encoded_string = base64.b64encode(image_file.read()).decode('utf-8') return {"image_base64_content": encoded_string} else: return {"error": f"Unsupported file type for path: {file_path}"} except FileNotFoundError: return {"error": f"File not found at path: {file_path}"} except Exception as e: return {"error": f"Failed to read file '{file_path}': {str(e)}"} whisper_model_cache = {} @tool def process_audio(file_path: str) -> dict: """ Transcribes an audio file (.mp3, .wav) using the OpenAI Whisper model. Use this when the user provides an audio file and asks for its content. Args: file_path: The local path to the audio file (e.g., 'validation/recipe.mp3'). """ try: # Import necessary libraries here to avoid loading them if the tool is not used from transformers import WhisperProcessor, WhisperForConditionalGeneration import librosa # Use a cache to avoid reloading the model on every call if "model" not in whisper_model_cache: print("--- Loading Whisper model for the first time... ---") processor = WhisperProcessor.from_pretrained("openai/whisper-small") model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-small") whisper_model_cache["processor"] = processor whisper_model_cache["model"] = model else: processor = whisper_model_cache["processor"] model = whisper_model_cache["model"] # Load audio file and resample to the required 16kHz audio_array, sampling_rate = librosa.load(file_path, sr=16000) # Process and transcribe input_features = processor(audio_array, sampling_rate=16000, return_tensors="pt").input_features predicted_ids = model.generate(input_features) transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True) return {"transcript": transcription[0]} except FileNotFoundError: return {"error": f"Audio file not found at path: {file_path}"} except Exception as e: return {"error": f"Failed to process audio file: {str(e)}"} @tool def read_python_file(file_path: str) -> dict: """ Reads the content of a Python (.py) file and returns it as a string. Use this tool FIRST when you are asked to analyze or determine the output of a Python script. Args: file_path: The local path to the Python file (e.g., 'validation/script.py'). """ try: # We already have robust file reading logic in `read_file`, let's reuse it! return read_file(file_path) except Exception as e: return {"error": str(e)} @tool def analyze_python_code(code: str) -> dict: """ Analyzes a string of Python code to determine its behavior and predict its final output. Does NOT execute the code. Use this AFTER you have read the code from a file. Args: code: A string containing the Python code to be analyzed. """ # This tool uses a separate, specific LLM call to analyze the code. # It's like asking a code expert to review it. try: # We will create a new, isolated LLM instance for this analysis # to prevent confusion with the main agent's state. analyzer_llm = ChatGoogleGenerativeAI( model="gemini-2.0-flash-lite", temperature=0, google_api_key=os.environ.get("GOOGLE_API_KEY") ) analysis_prompt = f""" You are an expert Python code analyst. Your task is to analyze the following code and determine its final numeric output. Think step-by-step. 1. Describe what the code does. 2. Trace the execution flow. 3. Pay close attention to loops, conditions, and random elements. 4. Determine the final value that will be printed. Here is the code: --- {code} --- Provide your analysis and then state the final output clearly. """ response = analyzer_llm.invoke(analysis_prompt) return {"code_analysis": response.content} except Exception as e: return {"error": f"Failed to analyze code: {str(e)}"} # @tool # def save_and_read_file(content: str, filename: Optional[str] = None) -> str: # """ # Save content to a file and return the path. # Args: # content (str): the content to save to the file # filename (str, optional): the name of the file. If not provided, a random name file will be created. # """ # temp_dir = tempfile.gettempdir() # if filename is None: # temp_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir) # filepath = temp_file.name # else: # filepath = os.path.join(temp_dir, filename) # with open(filepath, "w") as f: # f.write(content) # return f"File saved to {filepath}. You can read this file to process its contents." # @tool # def get_youtube_transcript(url: str) -> dict: # """ # Searches the web to find the transcript for a given YouTube URL. # This is a workaround for environments where direct YouTube access is blocked. # It uses a web search tool to find the transcript text. # Args: # url: The full URL of the YouTube video. # """ # try: # search_query = f"transcript for the youtube video {url}" # search_tool = TavilySearchResults(max_results=3) # search_results = search_tool.invoke({"query": search_query}) # if not search_results: # return {"error": "Web search returned no results for the transcript."} # transcript_content = "\n\n".join([doc.get("content", "") for doc in search_results]) # return { # "transcript_search_results": transcript_content[:4000] # Truncate to keep it manageable # } # except Exception as e: # return {"error": f"Failed to search for transcript using web search: {str(e)}"} # @tool # def download_file_from_url(url: str, filename: Optional[str] = None) -> str: # """ # Download a file from a URL and save it to a temporary location. # Args: # url (str): the URL of the file to download. # filename (str, optional): the name of the file. If not provided, a random name file will be created. # """ # try: # # Parse URL to get filename if not provided # if not filename: # path = urlparse(url).path # filename = os.path.basename(path) # if not filename: # filename = f"downloaded_{uuid.uuid4().hex[:8]}" # # Create temporary file # temp_dir = tempfile.gettempdir() # filepath = os.path.join(temp_dir, filename) # # Download the file # response = requests.get(url, stream=True) # response.raise_for_status() # # Save the file # with open(filepath, "wb") as f: # for chunk in response.iter_content(chunk_size=8192): # f.write(chunk) # return f"File downloaded to {filepath}. You can read this file to process its contents." # except Exception as e: # return f"Error downloading file: {str(e)}" # @tool # def extract_text_from_image(image_path: str) -> str: # """ # Extract text from an image using OCR library pytesseract (if available). # Args: # image_path (str): the path to the image file. # """ # try: # # Open the image # image = Image.open(image_path) # # Extract text from the image # text = pytesseract.image_to_string(image) # return f"Extracted text from image:\n\n{text}" # except Exception as e: # return f"Error extracting text from image: {str(e)}" # @tool # def analyze_csv_file(file_path: str, query: str) -> str: # """ # Analyze a CSV file using pandas and answer a question about it. # Args: # file_path (str): the path to the CSV file. # query (str): Question about the data # """ # try: # # Read the CSV file # df = pd.read_csv(file_path) # # Run various analyses based on the query # result = f"CSV file loaded with {len(df)} rows and {len(df.columns)} columns.\n" # result += f"Columns: {', '.join(df.columns)}\n\n" # # Add summary statistics # result += "Summary statistics:\n" # result += str(df.describe()) # return result # except Exception as e: # return f"Error analyzing CSV file: {str(e)}" # @tool # def analyze_excel_file(file_path: str, query: str) -> str: # """ # Analyze an Excel file using pandas and answer a question about it. # Args: # file_path (str): the path to the Excel file. # query (str): Question about the data # """ # try: # # Read the Excel file # df = pd.read_excel(file_path) # # Run various analyses based on the query # result = ( # f"Excel file loaded with {len(df)} rows and {len(df.columns)} columns.\n" # ) # result += f"Columns: {', '.join(df.columns)}\n\n" # # Add summary statistics # result += "Summary statistics:\n" # result += str(df.describe()) # return result # except Exception as e: # return f"Error analyzing Excel file: {str(e)}" # @tool # def analyze_image(image_base64: str) -> Dict[str, Any]: # """ # Analyze basic properties of an image (size, mode, color analysis, thumbnail preview). # Args: # image_base64 (str): Base64 encoded image string # Returns: # Dictionary with analysis result # """ # try: # img = decode_image(image_base64) # width, height = img.size # mode = img.mode # if mode in ("RGB", "RGBA"): # arr = np.array(img) # avg_colors = arr.mean(axis=(0, 1)) # dominant = ["Red", "Green", "Blue"][np.argmax(avg_colors[:3])] # brightness = avg_colors.mean() # color_analysis = { # "average_rgb": avg_colors.tolist(), # "brightness": brightness, # "dominant_color": dominant, # } # else: # color_analysis = {"note": f"No color analysis for mode {mode}"} # thumbnail = img.copy() # thumbnail.thumbnail((100, 100)) # thumb_path = save_image(thumbnail, "thumbnails") # thumbnail_base64 = encode_image(thumb_path) # return { # "dimensions": (width, height), # "mode": mode, # "color_analysis": color_analysis, # "thumbnail": thumbnail_base64, # } # except Exception as e: # return {"error": str(e)} # @tool # def transform_image( # image_base64: str, operation: str, params: Optional[Dict[str, Any]] = None # ) -> Dict[str, Any]: # """ # Apply transformations: resize, rotate, crop, flip, brightness, contrast, blur, sharpen, grayscale. # Args: # image_base64 (str): Base64 encoded input image # operation (str): Transformation operation # params (Dict[str, Any], optional): Parameters for the operation # Returns: # Dictionary with transformed image (base64) # """ # try: # img = decode_image(image_base64) # params = params or {} # if operation == "resize": # img = img.resize( # ( # params.get("width", img.width // 2), # params.get("height", img.height // 2), # ) # ) # elif operation == "rotate": # img = img.rotate(params.get("angle", 90), expand=True) # elif operation == "crop": # img = img.crop( # ( # params.get("left", 0), # params.get("top", 0), # params.get("right", img.width), # params.get("bottom", img.height), # ) # ) # elif operation == "flip": # if params.get("direction", "horizontal") == "horizontal": # img = img.transpose(Image.FLIP_LEFT_RIGHT) # else: # img = img.transpose(Image.FLIP_TOP_BOTTOM) # elif operation == "adjust_brightness": # img = ImageEnhance.Brightness(img).enhance(params.get("factor", 1.5)) # elif operation == "adjust_contrast": # img = ImageEnhance.Contrast(img).enhance(params.get("factor", 1.5)) # elif operation == "blur": # img = img.filter(ImageFilter.GaussianBlur(params.get("radius", 2))) # elif operation == "sharpen": # img = img.filter(ImageFilter.SHARPEN) # elif operation == "grayscale": # img = img.convert("L") # else: # return {"error": f"Unknown operation: {operation}"} # result_path = save_image(img) # result_base64 = encode_image(result_path) # return {"transformed_image": result_base64} # except Exception as e: # return {"error": str(e)} # Load system prompt with open("system_prompt.txt", "r", encoding = 'utf-8') as f: system_prompt = f.read() # Build system message sys_msg = SystemMessage(content = system_prompt) # Build a retriever embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") # dim=768 supabase: Client = create_client( os.environ.get('SUPABASE_URL'), os.environ.get('SUPABASE_SERVICE_KEY') ) vector_store = SupabaseVectorStore( client = supabase, embedding = embeddings, table_name = 'documents', query_name = 'match_document_langchain' ) create_retriever_tool = create_retriever_tool( retriever = vector_store.as_retriever(), name = "Question search", description = 'A tool to retrieve similar questions from a vector store' ) tools = [ add, subtract, multiply, divide, modulus, web_search, wiki_search, arxiv_search, fetch_arxiv_paper, read_file, read_python_file, analyze_python_code, process_audio # get_youtube_transcript, # download_file_from_url, save_and_read_file # extract_text_from_image, analyze_csv_file, analyze_excel_file, analyze_image, transform_image, ] # Build graph def build_graph(provider: str = 'google'): """ Build the graph using huggingface """ try: if provider == 'google': google_api_key = os.environ.get("GOOGLE_API_KEY") print("Setup API Key successfully") # Pass the key when you initialize the model llm_raw = ChatGoogleGenerativeAI( model="gemini-2.5-flash-lite", temperature=0.9, google_api_key=google_api_key # <-- Add this line ) else: llm_raw = ChatHuggingFace( llm = HuggingFaceEndpoint( repo_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0", task = 'text_generation', do_sample = False, temperature=0.9 ) ) except Exception as e: print(f"Error while loading model: {e}") llm = llm_raw.bind_tools(tools) # Nodes def assistant(state: MessagesState): """Assistant node""" return {"messages" : [llm.invoke(state["messages"])]} def retriever(state: MessagesState): """Retriever node""" similar_questions = vector_store.similarity_search(state["messages"][0].content) if similar_questions: example_msg = HumanMessage( content = f"This is a similar question and answer for references:\n\n{similar_questions[0].page_content}" ) return {"messages" : [sys_msg] + state['messages'] + [example_msg]} else: return {"messages" : [sys_msg] + state['messages']} builder = StateGraph(MessagesState) builder.add_node("retriever", retriever) builder.add_node("assistant", assistant) builder.add_node("tools", ToolNode(tools)) builder.add_edge(START, "retriever") builder.add_edge("retriever", "assistant") builder.add_conditional_edges("assistant", tools_condition) builder.add_edge("tools", "assistant") # builder = StateGraph(MessagesState) # builder.add_node("retriever", retriever) # # Retriever ist Start und Endpunkt # builder.set_entry_point("retriever") # builder.set_finish_point("retriever") return builder.compile() # if __name__ == "__main__": # question = "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect?" # graph = build_graph() # messages = [HumanMessage(content=question)] # messages = graph.invoke({"messages": messages}) # for m in messages["messages"]: # m.pretty_print()