Spaces:
Sleeping
Sleeping
Final_Assessment_PR
#1
by vallurimahanth - opened
- .huggingface.yaml +0 -1
- README.md +1 -1
- api_integration.py +0 -39
- app.py +3 -69
- core_agent.py +0 -523
- main.py +0 -257
- requirements.txt +1 -9
.huggingface.yaml
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
app_file: main.py
|
|
|
|
|
|
README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
emoji: 🕵🏻♂️
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: indigo
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Template Final Assignment
|
| 3 |
emoji: 🕵🏻♂️
|
| 4 |
colorFrom: indigo
|
| 5 |
colorTo: indigo
|
api_integration.py
DELETED
|
@@ -1,39 +0,0 @@
|
|
| 1 |
-
import requests
|
| 2 |
-
from typing import List, Dict, Any
|
| 3 |
-
from core_agent import GAIAAgent
|
| 4 |
-
|
| 5 |
-
class GAIAApiClient:
|
| 6 |
-
def __init__(self, api_url="https://agents-course-unit4-scoring.hf.space"):
|
| 7 |
-
self.api_url = api_url
|
| 8 |
-
self.questions_url = f"{api_url}/questions"
|
| 9 |
-
self.submit_url = f"{api_url}/submit"
|
| 10 |
-
self.files_url = f"{api_url}/files"
|
| 11 |
-
|
| 12 |
-
def get_questions(self) -> List[Dict[str, Any]]:
|
| 13 |
-
"""Fetch all evaluation questions"""
|
| 14 |
-
response = requests.get(self.questions_url)
|
| 15 |
-
response.raise_for_status()
|
| 16 |
-
return response.json()
|
| 17 |
-
|
| 18 |
-
def get_random_question(self) -> Dict[str, Any]:
|
| 19 |
-
"""Fetch a single random question"""
|
| 20 |
-
response = requests.get(f"{self.api_url}/random-question")
|
| 21 |
-
response.raise_for_status()
|
| 22 |
-
return response.json()
|
| 23 |
-
|
| 24 |
-
def get_file(self, task_id: str) -> bytes:
|
| 25 |
-
"""Download a file for a specific task"""
|
| 26 |
-
response = requests.get(f"{self.files_url}/{task_id}")
|
| 27 |
-
response.raise_for_status()
|
| 28 |
-
return response.content
|
| 29 |
-
|
| 30 |
-
def submit_answers(self, username: str, agent_code: str, answers: List[Dict[str, Any]]) -> Dict[str, Any]:
|
| 31 |
-
"""Submit agent answers and get score"""
|
| 32 |
-
data = {
|
| 33 |
-
"username": username,
|
| 34 |
-
"agent_code": agent_code,
|
| 35 |
-
"answers": answers
|
| 36 |
-
}
|
| 37 |
-
response = requests.post(self.submit_url, json=data)
|
| 38 |
-
response.raise_for_status()
|
| 39 |
-
return response.json()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app.py
CHANGED
|
@@ -3,28 +3,7 @@ import gradio as gr
|
|
| 3 |
import requests
|
| 4 |
import inspect
|
| 5 |
import pandas as pd
|
| 6 |
-
from core_agent import GAIAAgent
|
| 7 |
|
| 8 |
-
|
| 9 |
-
# Debug function to show available environment variables
|
| 10 |
-
def debug_environment():
|
| 11 |
-
"""Print available environment variables related to API keys (with values hidden)"""
|
| 12 |
-
debug_vars = [
|
| 13 |
-
"HF_API_TOKEN", "HUGGINGFACEHUB_API_TOKEN",
|
| 14 |
-
"OPENAI_API_KEY", "XAI_API_KEY",
|
| 15 |
-
"AGENT_MODEL_TYPE", "AGENT_MODEL_ID",
|
| 16 |
-
"AGENT_TEMPERATURE", "AGENT_VERBOSE"
|
| 17 |
-
]
|
| 18 |
-
|
| 19 |
-
print("=== DEBUG: Environment Variables ===")
|
| 20 |
-
for var in debug_vars:
|
| 21 |
-
if os.environ.get(var):
|
| 22 |
-
# Hide actual values for security
|
| 23 |
-
print(f"{var}: [SET]")
|
| 24 |
-
else:
|
| 25 |
-
print(f"{var}: [NOT SET]")
|
| 26 |
-
print("===================================")
|
| 27 |
-
|
| 28 |
# (Keep Constants as is)
|
| 29 |
# --- Constants ---
|
| 30 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
|
@@ -33,57 +12,12 @@ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
|
| 33 |
# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
|
| 34 |
class BasicAgent:
|
| 35 |
def __init__(self):
|
| 36 |
-
|
| 37 |
print("BasicAgent initialized.")
|
| 38 |
-
debug_environment()
|
| 39 |
-
try:
|
| 40 |
-
try:
|
| 41 |
-
import dotenv
|
| 42 |
-
dotenv.load_dotenv()
|
| 43 |
-
print("Loaded environment variables from .env file")
|
| 44 |
-
except ImportError:
|
| 45 |
-
print("python-dotenv not installed, continuing with environment as is")
|
| 46 |
-
|
| 47 |
-
hf_token = os.environ.get("HF_API_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
|
| 48 |
-
temperature = float(os.environ.get("AGENT_TEMPERATURE", "0.2"))
|
| 49 |
-
verbose = os.environ.get("AGENT_VERBOSE", "false").lower() == "true"
|
| 50 |
-
|
| 51 |
-
if hf_token:
|
| 52 |
-
self.gaia_agent = GAIAAgent(
|
| 53 |
-
model_type="HfApiModel",
|
| 54 |
-
model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
|
| 55 |
-
api_key=hf_token,
|
| 56 |
-
temperature=temperature,
|
| 57 |
-
executor_type="local",
|
| 58 |
-
verbose=verbose
|
| 59 |
-
)
|
| 60 |
-
else:
|
| 61 |
-
print("ERROR: No API keys found. Please set at least one of these environment variables:")
|
| 62 |
-
self.gaia_agent = None
|
| 63 |
-
print("WARNING: No API keys found. Agent will not be able to answer questions.")
|
| 64 |
-
except Exception as e:
|
| 65 |
-
print(f"Error initializing GAIAAgent: {e}")
|
| 66 |
-
self.gaia_agent = None
|
| 67 |
-
print("WARNING: Failed to initialize agent. Falling back to basic responses.")
|
| 68 |
-
|
| 69 |
-
|
| 70 |
def __call__(self, question: str) -> str:
|
| 71 |
print(f"Agent received question (first 50 chars): {question[:50]}...")
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
# Process the question using the GAIA agent
|
| 76 |
-
answer = self.gaia_agent.answer_question(question)
|
| 77 |
-
print(f"Agent generated answer: {answer[:50]}..." if len(answer) > 50 else f"Agent generated answer: {answer}")
|
| 78 |
-
return answer
|
| 79 |
-
except Exception as e:
|
| 80 |
-
print(f"Error processing question: {e}")
|
| 81 |
-
# Fall back to a simple response on error
|
| 82 |
-
return "An error occurred while processing your question. Please check the agent logs for details."
|
| 83 |
-
else:
|
| 84 |
-
# We don't have a valid agent, provide a basic response
|
| 85 |
-
return "The agent is not properly initialized. Please check your API keys and configuration."
|
| 86 |
-
|
| 87 |
|
| 88 |
def run_and_submit_all( profile: gr.OAuthProfile | None):
|
| 89 |
"""
|
|
|
|
| 3 |
import requests
|
| 4 |
import inspect
|
| 5 |
import pandas as pd
|
|
|
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
# (Keep Constants as is)
|
| 8 |
# --- Constants ---
|
| 9 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
|
|
|
| 12 |
# ----- THIS IS WERE YOU CAN BUILD WHAT YOU WANT ------
|
| 13 |
class BasicAgent:
|
| 14 |
def __init__(self):
|
|
|
|
| 15 |
print("BasicAgent initialized.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def __call__(self, question: str) -> str:
|
| 17 |
print(f"Agent received question (first 50 chars): {question[:50]}...")
|
| 18 |
+
fixed_answer = "This is a default answer."
|
| 19 |
+
print(f"Agent returning fixed answer: {fixed_answer}")
|
| 20 |
+
return fixed_answer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
def run_and_submit_all( profile: gr.OAuthProfile | None):
|
| 23 |
"""
|
core_agent.py
DELETED
|
@@ -1,523 +0,0 @@
|
|
| 1 |
-
from smolagents import (
|
| 2 |
-
CodeAgent,
|
| 3 |
-
DuckDuckGoSearchTool,
|
| 4 |
-
HfApiModel,
|
| 5 |
-
LiteLLMModel,
|
| 6 |
-
OpenAIServerModel,
|
| 7 |
-
PythonInterpreterTool,
|
| 8 |
-
tool,
|
| 9 |
-
InferenceClientModel,
|
| 10 |
-
GoogleSearchTool,
|
| 11 |
-
VisitWebpageTool
|
| 12 |
-
)
|
| 13 |
-
from typing import List, Dict, Any, Optional
|
| 14 |
-
import os
|
| 15 |
-
import tempfile
|
| 16 |
-
import re
|
| 17 |
-
import json
|
| 18 |
-
import requests
|
| 19 |
-
from urllib.parse import urlparse
|
| 20 |
-
# Load model directly
|
| 21 |
-
|
| 22 |
-
@tool
|
| 23 |
-
def save_and_read_file(content: str, filename: Optional[str] = None) -> str:
|
| 24 |
-
"""
|
| 25 |
-
Save content to a temporary file and return the path.
|
| 26 |
-
Useful for processing files from the GAIA API.
|
| 27 |
-
|
| 28 |
-
Args:
|
| 29 |
-
content: The content to save to the file
|
| 30 |
-
filename: Optional filename, will generate a random name if not provided
|
| 31 |
-
|
| 32 |
-
Returns:
|
| 33 |
-
Path to the saved file
|
| 34 |
-
"""
|
| 35 |
-
temp_dir = tempfile.gettempdir()
|
| 36 |
-
if filename is None:
|
| 37 |
-
temp_file = tempfile.NamedTemporaryFile(delete=False)
|
| 38 |
-
filepath = temp_file.name
|
| 39 |
-
else:
|
| 40 |
-
filepath = os.path.join(temp_dir, filename)
|
| 41 |
-
|
| 42 |
-
# Write content to the file
|
| 43 |
-
with open(filepath, 'w') as f:
|
| 44 |
-
f.write(content)
|
| 45 |
-
|
| 46 |
-
return f"File saved to {filepath}. You can read this file to process its contents."
|
| 47 |
-
|
| 48 |
-
@tool
|
| 49 |
-
def download_file_from_url(url: str, filename: Optional[str] = None) -> str:
|
| 50 |
-
"""
|
| 51 |
-
Download a file from a URL and save it to a temporary location.
|
| 52 |
-
|
| 53 |
-
Args:
|
| 54 |
-
url: The URL to download from
|
| 55 |
-
filename: Optional filename, will generate one based on URL if not provided
|
| 56 |
-
|
| 57 |
-
Returns:
|
| 58 |
-
Path to the downloaded file
|
| 59 |
-
"""
|
| 60 |
-
try:
|
| 61 |
-
# Parse URL to get filename if not provided
|
| 62 |
-
if not filename:
|
| 63 |
-
path = urlparse(url).path
|
| 64 |
-
filename = os.path.basename(path)
|
| 65 |
-
if not filename:
|
| 66 |
-
# Generate a random name if we couldn't extract one
|
| 67 |
-
import uuid
|
| 68 |
-
filename = f"downloaded_{uuid.uuid4().hex[:8]}"
|
| 69 |
-
|
| 70 |
-
# Create temporary file
|
| 71 |
-
temp_dir = tempfile.gettempdir()
|
| 72 |
-
filepath = os.path.join(temp_dir, filename)
|
| 73 |
-
|
| 74 |
-
# Download the file
|
| 75 |
-
response = requests.get(url, stream=True)
|
| 76 |
-
response.raise_for_status()
|
| 77 |
-
|
| 78 |
-
# Save the file
|
| 79 |
-
with open(filepath, 'wb') as f:
|
| 80 |
-
for chunk in response.iter_content(chunk_size=8192):
|
| 81 |
-
f.write(chunk)
|
| 82 |
-
|
| 83 |
-
return f"File downloaded to {filepath}. You can now process this file."
|
| 84 |
-
except Exception as e:
|
| 85 |
-
return f"Error downloading file: {str(e)}"
|
| 86 |
-
|
| 87 |
-
@tool
|
| 88 |
-
def extract_text_from_image(image_path: str) -> str:
|
| 89 |
-
"""
|
| 90 |
-
Extract text from an image using pytesseract (if available).
|
| 91 |
-
|
| 92 |
-
Args:
|
| 93 |
-
image_path: Path to the image file
|
| 94 |
-
|
| 95 |
-
Returns:
|
| 96 |
-
Extracted text or error message
|
| 97 |
-
"""
|
| 98 |
-
try:
|
| 99 |
-
# Try to import pytesseract
|
| 100 |
-
import pytesseract
|
| 101 |
-
from PIL import Image
|
| 102 |
-
|
| 103 |
-
# Open the image
|
| 104 |
-
image = Image.open(image_path)
|
| 105 |
-
|
| 106 |
-
# Extract text
|
| 107 |
-
text = pytesseract.image_to_string(image)
|
| 108 |
-
|
| 109 |
-
return f"Extracted text from image:\n\n{text}"
|
| 110 |
-
except ImportError:
|
| 111 |
-
return "Error: pytesseract is not installed. Please install it with 'pip install pytesseract' and ensure Tesseract OCR is installed on your system."
|
| 112 |
-
except Exception as e:
|
| 113 |
-
return f"Error extracting text from image: {str(e)}"
|
| 114 |
-
|
| 115 |
-
@tool
|
| 116 |
-
def check_vegetables_from_list(list_of_items: List[str]) -> List[str]:
|
| 117 |
-
"""
|
| 118 |
-
Use this function whenever you need to check the vegetables in a list of items.
|
| 119 |
-
This function takes input as a list of items and returns a list of vegetables extracted from the given list of items.
|
| 120 |
-
|
| 121 |
-
Args:
|
| 122 |
-
list_of_items: This is a list of items from which you will check what are the vegetables in it.
|
| 123 |
-
|
| 124 |
-
Returns:
|
| 125 |
-
Extracted List of Vegetables or empty list
|
| 126 |
-
"""
|
| 127 |
-
try:
|
| 128 |
-
print(list_of_items)
|
| 129 |
-
vegetables=["broccoli", "celery", "basil", "lettuce", "sweet potatoes"]
|
| 130 |
-
result = []
|
| 131 |
-
for item in list_of_items:
|
| 132 |
-
for veg in vegetables:
|
| 133 |
-
if veg in item:
|
| 134 |
-
result.append(item)
|
| 135 |
-
break
|
| 136 |
-
|
| 137 |
-
result.sort()
|
| 138 |
-
checkingOutputLog=f"Extracted List of Vegetables: \n\n{result}"
|
| 139 |
-
print(checkingOutputLog)
|
| 140 |
-
return result
|
| 141 |
-
except Exception as e:
|
| 142 |
-
return [str(e)]
|
| 143 |
-
|
| 144 |
-
@tool
|
| 145 |
-
def analyze_csv_file(file_path: str, query: str) -> str:
|
| 146 |
-
"""
|
| 147 |
-
Analyze a CSV file using pandas and answer a question about it.
|
| 148 |
-
|
| 149 |
-
Args:
|
| 150 |
-
file_path: Path to the CSV file
|
| 151 |
-
query: Question about the data
|
| 152 |
-
|
| 153 |
-
Returns:
|
| 154 |
-
Analysis result or error message
|
| 155 |
-
"""
|
| 156 |
-
try:
|
| 157 |
-
import pandas as pd
|
| 158 |
-
|
| 159 |
-
# Read the CSV file
|
| 160 |
-
df = pd.read_csv(file_path)
|
| 161 |
-
|
| 162 |
-
# Run various analyses based on the query
|
| 163 |
-
result = f"CSV file loaded with {len(df)} rows and {len(df.columns)} columns.\n"
|
| 164 |
-
result += f"Columns: {', '.join(df.columns)}\n\n"
|
| 165 |
-
|
| 166 |
-
# Add summary statistics
|
| 167 |
-
result += "Summary statistics:\n"
|
| 168 |
-
result += str(df.describe())
|
| 169 |
-
|
| 170 |
-
return result
|
| 171 |
-
except ImportError:
|
| 172 |
-
return "Error: pandas is not installed. Please install it with 'pip install pandas'."
|
| 173 |
-
except Exception as e:
|
| 174 |
-
return f"Error analyzing CSV file: {str(e)}"
|
| 175 |
-
|
| 176 |
-
@tool
|
| 177 |
-
def analyze_excel_file(file_path: str, query: str) -> str:
|
| 178 |
-
"""
|
| 179 |
-
Analyze an Excel file using pandas and answer a question about it.
|
| 180 |
-
|
| 181 |
-
Args:
|
| 182 |
-
file_path: Path to the Excel file
|
| 183 |
-
query: Question about the data
|
| 184 |
-
|
| 185 |
-
Returns:
|
| 186 |
-
Analysis result or error message
|
| 187 |
-
"""
|
| 188 |
-
try:
|
| 189 |
-
import pandas as pd
|
| 190 |
-
|
| 191 |
-
# Read the Excel file
|
| 192 |
-
df = pd.read_excel(file_path)
|
| 193 |
-
|
| 194 |
-
# Run various analyses based on the query
|
| 195 |
-
result = f"Excel file loaded with {len(df)} rows and {len(df.columns)} columns.\n"
|
| 196 |
-
result += f"Columns: {', '.join(df.columns)}\n\n"
|
| 197 |
-
|
| 198 |
-
# Add summary statistics
|
| 199 |
-
result += "Summary statistics:\n"
|
| 200 |
-
result += str(df.describe())
|
| 201 |
-
|
| 202 |
-
return result
|
| 203 |
-
except ImportError:
|
| 204 |
-
return "Error: pandas and openpyxl are not installed. Please install them with 'pip install pandas openpyxl'."
|
| 205 |
-
except Exception as e:
|
| 206 |
-
return f"Error analyzing Excel file: {str(e)}"
|
| 207 |
-
|
| 208 |
-
class GAIAAgent:
|
| 209 |
-
def __init__(
|
| 210 |
-
self,
|
| 211 |
-
model_type: str = "HfApiModel",
|
| 212 |
-
model_id: Optional[str] = None,
|
| 213 |
-
api_key: Optional[str] = None,
|
| 214 |
-
api_base: Optional[str] = None,
|
| 215 |
-
temperature: float = 0.1,
|
| 216 |
-
executor_type: str = "local", # Changed from use_e2b to executor_type
|
| 217 |
-
additional_imports: List[str] = None,
|
| 218 |
-
additional_tools: List[Any] = None,
|
| 219 |
-
system_prompt: Optional[str] = None, # We'll still accept this parameter but not use it directly
|
| 220 |
-
verbose: bool = False,
|
| 221 |
-
provider: Optional[str] = None, # Add provider for InferenceClientModel
|
| 222 |
-
timeout: Optional[int] = None # Add timeout for InferenceClientModel
|
| 223 |
-
):
|
| 224 |
-
"""
|
| 225 |
-
Initialize a GAIAAgent with specified configuration
|
| 226 |
-
|
| 227 |
-
Args:
|
| 228 |
-
model_type: Type of model to use (HfApiModel, LiteLLMModel, OpenAIServerModel, InferenceClientModel)
|
| 229 |
-
model_id: ID of the model to use
|
| 230 |
-
api_key: API key for the model provider
|
| 231 |
-
api_base: Base URL for API calls
|
| 232 |
-
temperature: Temperature for text generation
|
| 233 |
-
executor_type: Type of executor for code execution ('local' or 'e2b')
|
| 234 |
-
additional_imports: Additional Python modules to allow importing
|
| 235 |
-
additional_tools: Additional tools to provide to the agent
|
| 236 |
-
system_prompt: Custom system prompt to use (not directly used, kept for backward compatibility)
|
| 237 |
-
verbose: Enable verbose logging
|
| 238 |
-
provider: Provider for InferenceClientModel (e.g., "hf-inference")
|
| 239 |
-
timeout: Timeout in seconds for API calls
|
| 240 |
-
"""
|
| 241 |
-
# Set verbosity
|
| 242 |
-
self.verbose = verbose
|
| 243 |
-
self.system_prompt = system_prompt # Store for potential future use
|
| 244 |
-
|
| 245 |
-
# Initialize model based on configuration
|
| 246 |
-
if model_type == "HfApiModel":
|
| 247 |
-
if api_key is None:
|
| 248 |
-
api_key = os.getenv("HUGGINGFACEHUB_API_TOKEN")
|
| 249 |
-
if not api_key:
|
| 250 |
-
raise ValueError("No Hugging Face token provided. Please set HUGGINGFACEHUB_API_TOKEN environment variable or pass api_key parameter.")
|
| 251 |
-
|
| 252 |
-
if self.verbose:
|
| 253 |
-
print(f"Using Hugging Face token: {api_key[:5]}...")
|
| 254 |
-
|
| 255 |
-
self.model = HfApiModel(
|
| 256 |
-
model_id=model_id or "meta-llama/Llama-3-70B-Instruct",
|
| 257 |
-
token=api_key,
|
| 258 |
-
temperature=temperature
|
| 259 |
-
)
|
| 260 |
-
elif model_type == "InferenceClientModel":
|
| 261 |
-
if api_key is None:
|
| 262 |
-
api_key = os.getenv("HUGGINGFACEHUB_API_TOKEN")
|
| 263 |
-
if not api_key:
|
| 264 |
-
raise ValueError("No Hugging Face token provided. Please set HUGGINGFACEHUB_API_TOKEN environment variable or pass api_key parameter.")
|
| 265 |
-
|
| 266 |
-
if self.verbose:
|
| 267 |
-
print(f"Using Hugging Face token: {api_key[:5]}...")
|
| 268 |
-
|
| 269 |
-
self.model = InferenceClientModel(
|
| 270 |
-
model_id=model_id or "meta-llama/Llama-3-70B-Instruct",
|
| 271 |
-
provider=provider or "hf-inference",
|
| 272 |
-
token=api_key,
|
| 273 |
-
timeout=timeout or 120,
|
| 274 |
-
temperature=temperature
|
| 275 |
-
)
|
| 276 |
-
elif model_type == "LiteLLMModel":
|
| 277 |
-
from smolagents import LiteLLMModel
|
| 278 |
-
self.model = LiteLLMModel(
|
| 279 |
-
model_id=model_id or "gpt-4o",
|
| 280 |
-
api_key=api_key or os.getenv("OPENAI_API_KEY"),
|
| 281 |
-
temperature=temperature
|
| 282 |
-
)
|
| 283 |
-
elif model_type == "OpenAIServerModel":
|
| 284 |
-
# Check for xAI API key and base URL first
|
| 285 |
-
xai_api_key = os.getenv("XAI_API_KEY")
|
| 286 |
-
xai_api_base = os.getenv("XAI_API_BASE")
|
| 287 |
-
|
| 288 |
-
# If xAI credentials are available, use them
|
| 289 |
-
if xai_api_key and api_key is None:
|
| 290 |
-
api_key = xai_api_key
|
| 291 |
-
if self.verbose:
|
| 292 |
-
print(f"Using xAI API key: {api_key[:5]}...")
|
| 293 |
-
|
| 294 |
-
# If no API key specified, fall back to OPENAI_API_KEY
|
| 295 |
-
if api_key is None:
|
| 296 |
-
api_key = os.getenv("OPENAI_API_KEY")
|
| 297 |
-
if not api_key:
|
| 298 |
-
raise ValueError("No OpenAI API key provided. Please set OPENAI_API_KEY or XAI_API_KEY environment variable or pass api_key parameter.")
|
| 299 |
-
|
| 300 |
-
# If xAI API base is available and no api_base is provided, use it
|
| 301 |
-
if xai_api_base and api_base is None:
|
| 302 |
-
api_base = xai_api_base
|
| 303 |
-
if self.verbose:
|
| 304 |
-
print(f"Using xAI API base URL: {api_base}")
|
| 305 |
-
|
| 306 |
-
# If no API base specified but environment variable available, use it
|
| 307 |
-
if api_base is None:
|
| 308 |
-
api_base = os.getenv("AGENT_API_BASE")
|
| 309 |
-
if api_base and self.verbose:
|
| 310 |
-
print(f"Using API base from AGENT_API_BASE: {api_base}")
|
| 311 |
-
|
| 312 |
-
self.model = OpenAIServerModel(
|
| 313 |
-
model_id=model_id or "gpt-4o",
|
| 314 |
-
api_key=api_key,
|
| 315 |
-
api_base=api_base,
|
| 316 |
-
temperature=temperature
|
| 317 |
-
)
|
| 318 |
-
else:
|
| 319 |
-
raise ValueError(f"Unknown model type: {model_type}")
|
| 320 |
-
|
| 321 |
-
if self.verbose:
|
| 322 |
-
print(f"Initialized model: {model_type} - {model_id}")
|
| 323 |
-
|
| 324 |
-
# Initialize default tools
|
| 325 |
-
self.tools = [
|
| 326 |
-
GoogleSearchTool("serper"),
|
| 327 |
-
VisitWebpageTool(),
|
| 328 |
-
PythonInterpreterTool(),
|
| 329 |
-
check_vegetables_from_list,
|
| 330 |
-
save_and_read_file,
|
| 331 |
-
download_file_from_url,
|
| 332 |
-
analyze_csv_file,
|
| 333 |
-
analyze_excel_file
|
| 334 |
-
]
|
| 335 |
-
|
| 336 |
-
# Add extract_text_from_image if PIL and pytesseract are available
|
| 337 |
-
try:
|
| 338 |
-
import pytesseract
|
| 339 |
-
from PIL import Image
|
| 340 |
-
self.tools.append(extract_text_from_image)
|
| 341 |
-
if self.verbose:
|
| 342 |
-
print("Added image processing tool")
|
| 343 |
-
except ImportError:
|
| 344 |
-
if self.verbose:
|
| 345 |
-
print("Image processing libraries not available")
|
| 346 |
-
|
| 347 |
-
# Add any additional tools
|
| 348 |
-
if additional_tools:
|
| 349 |
-
self.tools.extend(additional_tools)
|
| 350 |
-
|
| 351 |
-
if self.verbose:
|
| 352 |
-
print(f"Initialized with {len(self.tools)} tools")
|
| 353 |
-
|
| 354 |
-
# Setup imports allowed
|
| 355 |
-
self.imports = ["pandas", "numpy", "datetime", "json", "re", "math", "os", "requests", "csv", "urllib"]
|
| 356 |
-
if additional_imports:
|
| 357 |
-
self.imports.extend(additional_imports)
|
| 358 |
-
|
| 359 |
-
# Initialize the CodeAgent
|
| 360 |
-
executor_kwargs = {}
|
| 361 |
-
if executor_type == "e2b":
|
| 362 |
-
try:
|
| 363 |
-
# Try to import e2b dependencies to check if they're available
|
| 364 |
-
from e2b_code_interpreter import Sandbox
|
| 365 |
-
if self.verbose:
|
| 366 |
-
print("Using e2b executor")
|
| 367 |
-
except ImportError:
|
| 368 |
-
if self.verbose:
|
| 369 |
-
print("e2b dependencies not found, falling back to local executor")
|
| 370 |
-
executor_type = "local" # Fallback to local if e2b is not available
|
| 371 |
-
|
| 372 |
-
self.agent = CodeAgent(
|
| 373 |
-
tools=self.tools,
|
| 374 |
-
model=self.model,
|
| 375 |
-
additional_authorized_imports=self.imports,
|
| 376 |
-
executor_type=executor_type,
|
| 377 |
-
executor_kwargs=executor_kwargs,
|
| 378 |
-
verbosity_level=2 if self.verbose else 0
|
| 379 |
-
)
|
| 380 |
-
|
| 381 |
-
if self.verbose:
|
| 382 |
-
print("Agent initialized and ready")
|
| 383 |
-
|
| 384 |
-
def answer_question(self, question: str, task_file_path: Optional[str] = None) -> str:
|
| 385 |
-
"""
|
| 386 |
-
Process a GAIA benchmark question and return the answer
|
| 387 |
-
|
| 388 |
-
Args:
|
| 389 |
-
question: The question to answer
|
| 390 |
-
task_file_path: Optional path to a file associated with the question
|
| 391 |
-
|
| 392 |
-
Returns:
|
| 393 |
-
The answer to the question
|
| 394 |
-
"""
|
| 395 |
-
try:
|
| 396 |
-
if self.verbose:
|
| 397 |
-
print(f"Processing question: {question}")
|
| 398 |
-
if task_file_path:
|
| 399 |
-
print(f"With associated file: {task_file_path}")
|
| 400 |
-
|
| 401 |
-
# Create a context with file information if available
|
| 402 |
-
context = question
|
| 403 |
-
file_content = None
|
| 404 |
-
|
| 405 |
-
# If there's a file, read it and include its content in the context
|
| 406 |
-
if task_file_path:
|
| 407 |
-
try:
|
| 408 |
-
with open(task_file_path, 'r') as f:
|
| 409 |
-
file_content = f.read()
|
| 410 |
-
|
| 411 |
-
# Determine file type from extension
|
| 412 |
-
import os
|
| 413 |
-
file_ext = os.path.splitext(task_file_path)[1].lower()
|
| 414 |
-
|
| 415 |
-
context = f"""
|
| 416 |
-
Question: {question}
|
| 417 |
-
This question has an associated file. Here is the file content:
|
| 418 |
-
```{file_ext}
|
| 419 |
-
{file_content}
|
| 420 |
-
```
|
| 421 |
-
Analyze the file content above to answer the question.
|
| 422 |
-
"""
|
| 423 |
-
except Exception as file_e:
|
| 424 |
-
context = f"""
|
| 425 |
-
Question: {question}
|
| 426 |
-
This question has an associated file at path: {task_file_path}
|
| 427 |
-
However, there was an error reading the file: {file_e}
|
| 428 |
-
You can still try to answer the question based on the information provided.
|
| 429 |
-
"""
|
| 430 |
-
|
| 431 |
-
# Check for special cases that need specific formatting
|
| 432 |
-
# Reversed text questions
|
| 433 |
-
if question.startswith(".") or ".rewsna eht sa" in question:
|
| 434 |
-
context = f"""
|
| 435 |
-
This question appears to be in reversed text. Here's the reversed version:
|
| 436 |
-
{question[::-1]}
|
| 437 |
-
Now answer the question above. Remember to format your answer exactly as requested.
|
| 438 |
-
"""
|
| 439 |
-
if "basil" in question:
|
| 440 |
-
context=f"""{context} This question is related to vegetables. For checking the vegetables you have to use the custom check_vegetables_from_list tool and
|
| 441 |
-
get the extracted list of vegetables."""
|
| 442 |
-
|
| 443 |
-
# Add a prompt to ensure precise answers
|
| 444 |
-
full_prompt = f"""{context}
|
| 445 |
-
When answering, provide ONLY the precise answer requested.
|
| 446 |
-
Do not include explanations, steps, reasoning, or additional text.
|
| 447 |
-
Be direct and specific. GAIA benchmark requires exact matching answers.
|
| 448 |
-
For example, if asked "What is the capital of France?", respond simply with "Paris".
|
| 449 |
-
"""
|
| 450 |
-
|
| 451 |
-
# Run the agent with the question
|
| 452 |
-
answer = self.agent.run(full_prompt)
|
| 453 |
-
|
| 454 |
-
# Clean up the answer to ensure it's in the expected format
|
| 455 |
-
# Remove common prefixes that models often add
|
| 456 |
-
answer = self._clean_answer(answer)
|
| 457 |
-
|
| 458 |
-
if self.verbose:
|
| 459 |
-
print(f"Generated answer: {answer}")
|
| 460 |
-
|
| 461 |
-
return answer
|
| 462 |
-
except Exception as e:
|
| 463 |
-
error_msg = f"Error answering question: {e}"
|
| 464 |
-
if self.verbose:
|
| 465 |
-
print(error_msg)
|
| 466 |
-
return error_msg
|
| 467 |
-
|
| 468 |
-
def _clean_answer(self, answer: any) -> str:
|
| 469 |
-
"""
|
| 470 |
-
Clean up the answer to remove common prefixes and formatting
|
| 471 |
-
that models often add but that can cause exact match failures.
|
| 472 |
-
|
| 473 |
-
Args:
|
| 474 |
-
answer: The raw answer from the model
|
| 475 |
-
|
| 476 |
-
Returns:
|
| 477 |
-
The cleaned answer as a string
|
| 478 |
-
"""
|
| 479 |
-
# Convert non-string types to strings
|
| 480 |
-
if not isinstance(answer, str):
|
| 481 |
-
# Handle numeric types (float, int)
|
| 482 |
-
if isinstance(answer, float):
|
| 483 |
-
# Format floating point numbers properly
|
| 484 |
-
# Check if it's an integer value in float form (e.g., 12.0)
|
| 485 |
-
if answer.is_integer():
|
| 486 |
-
formatted_answer = str(int(answer))
|
| 487 |
-
else:
|
| 488 |
-
# For currency values that might need formatting
|
| 489 |
-
if abs(answer) >= 1000:
|
| 490 |
-
formatted_answer = f"${answer:,.2f}"
|
| 491 |
-
else:
|
| 492 |
-
formatted_answer = str(answer)
|
| 493 |
-
return formatted_answer
|
| 494 |
-
elif isinstance(answer, int):
|
| 495 |
-
return str(answer)
|
| 496 |
-
else:
|
| 497 |
-
# For any other type
|
| 498 |
-
return str(answer)
|
| 499 |
-
|
| 500 |
-
# Now we know answer is a string, so we can safely use string methods
|
| 501 |
-
# Normalize whitespace
|
| 502 |
-
answer = answer.strip()
|
| 503 |
-
|
| 504 |
-
# Remove common prefixes and formatting that models add
|
| 505 |
-
prefixes_to_remove = [
|
| 506 |
-
"The answer is ",
|
| 507 |
-
"Answer: ",
|
| 508 |
-
"Final answer: ",
|
| 509 |
-
"The result is ",
|
| 510 |
-
"To answer this question: ",
|
| 511 |
-
"Based on the information provided, ",
|
| 512 |
-
"According to the information: ",
|
| 513 |
-
]
|
| 514 |
-
|
| 515 |
-
for prefix in prefixes_to_remove:
|
| 516 |
-
if answer.startswith(prefix):
|
| 517 |
-
answer = answer[len(prefix):].strip()
|
| 518 |
-
|
| 519 |
-
# Remove quotes if they wrap the entire answer
|
| 520 |
-
if (answer.startswith('"') and answer.endswith('"')) or (answer.startswith("'") and answer.endswith("'")):
|
| 521 |
-
answer = answer[1:-1].strip()
|
| 522 |
-
|
| 523 |
-
return answer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
main.py
DELETED
|
@@ -1,257 +0,0 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import tempfile
|
| 3 |
-
import gradio as gr
|
| 4 |
-
import pandas as pd
|
| 5 |
-
import traceback
|
| 6 |
-
from core_agent import GAIAAgent
|
| 7 |
-
from api_integration import GAIAApiClient
|
| 8 |
-
|
| 9 |
-
# Constants
|
| 10 |
-
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 11 |
-
|
| 12 |
-
def save_task_file(file_content, task_id):
|
| 13 |
-
"""
|
| 14 |
-
Save a task file to a temporary location
|
| 15 |
-
"""
|
| 16 |
-
if not file_content:
|
| 17 |
-
return None
|
| 18 |
-
|
| 19 |
-
# Create a temporary file
|
| 20 |
-
temp_dir = tempfile.gettempdir()
|
| 21 |
-
file_path = os.path.join(temp_dir, f"gaia_task_{task_id}.txt")
|
| 22 |
-
|
| 23 |
-
# Write content to the file
|
| 24 |
-
with open(file_path, 'wb') as f:
|
| 25 |
-
f.write(file_content)
|
| 26 |
-
|
| 27 |
-
print(f"File saved to {file_path}")
|
| 28 |
-
return file_path
|
| 29 |
-
|
| 30 |
-
def get_agent_configuration():
|
| 31 |
-
"""
|
| 32 |
-
Get the agent configuration based on environment variables
|
| 33 |
-
"""
|
| 34 |
-
# Default configuration
|
| 35 |
-
config = {
|
| 36 |
-
"model_type": "OpenAIServerModel", # Default to OpenAIServerModel
|
| 37 |
-
"model_id": "gpt-4o", # Default model for OpenAI
|
| 38 |
-
"temperature": 0.1,
|
| 39 |
-
"executor_type": "local",
|
| 40 |
-
"verbose": False,
|
| 41 |
-
"provider": "hf-inference", # For InferenceClientModel
|
| 42 |
-
"timeout": 120 # For InferenceClientModel
|
| 43 |
-
}
|
| 44 |
-
|
| 45 |
-
# Check for xAI API key and base URL
|
| 46 |
-
xai_api_key = os.getenv("XAI_API_KEY")
|
| 47 |
-
xai_api_base = os.getenv("XAI_API_BASE")
|
| 48 |
-
|
| 49 |
-
# If we have xAI credentials, use them
|
| 50 |
-
if xai_api_key:
|
| 51 |
-
config["api_key"] = xai_api_key
|
| 52 |
-
if xai_api_base:
|
| 53 |
-
config["api_base"] = xai_api_base
|
| 54 |
-
|
| 55 |
-
return config
|
| 56 |
-
|
| 57 |
-
def run_and_submit_all(profile: gr.OAuthProfile | None):
|
| 58 |
-
"""
|
| 59 |
-
Fetches all questions, runs the GAIAAgent on them, submits all answers,
|
| 60 |
-
and displays the results.
|
| 61 |
-
"""
|
| 62 |
-
# Check for user login
|
| 63 |
-
if not profile:
|
| 64 |
-
return "Please Login to Hugging Face with the button.", None
|
| 65 |
-
|
| 66 |
-
username = profile.username
|
| 67 |
-
print(f"User logged in: {username}")
|
| 68 |
-
|
| 69 |
-
# Get SPACE_ID for code link
|
| 70 |
-
space_id = os.getenv("SPACE_ID")
|
| 71 |
-
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
|
| 72 |
-
|
| 73 |
-
# Initialize API client
|
| 74 |
-
api_client = GAIAApiClient(DEFAULT_API_URL)
|
| 75 |
-
|
| 76 |
-
# Initialize Agent with configuration
|
| 77 |
-
try:
|
| 78 |
-
agent_config = get_agent_configuration()
|
| 79 |
-
print(f"Using agent configuration: {agent_config}")
|
| 80 |
-
|
| 81 |
-
agent = GAIAAgent(**agent_config)
|
| 82 |
-
print("Agent initialized successfully")
|
| 83 |
-
except Exception as e:
|
| 84 |
-
error_details = traceback.format_exc()
|
| 85 |
-
print(f"Error initializing agent: {e}\n{error_details}")
|
| 86 |
-
return f"Error initializing agent: {e}", None
|
| 87 |
-
|
| 88 |
-
# Fetch questions
|
| 89 |
-
try:
|
| 90 |
-
questions_data_v2 = api_client.get_questions()
|
| 91 |
-
questions_data=[]
|
| 92 |
-
questions_data.append(questions_data_v2[2])
|
| 93 |
-
questions_data.append(questions_data_v2[4])
|
| 94 |
-
questions_data.append(questions_data_v2[7])
|
| 95 |
-
questions_data.append(questions_data_v2[8])
|
| 96 |
-
questions_data.append(questions_data_v2[11])
|
| 97 |
-
questions_data.append(questions_data_v2[14])
|
| 98 |
-
questions_data.append(questions_data_v2[19])
|
| 99 |
-
if not questions_data:
|
| 100 |
-
return "Fetched questions list is empty or invalid format.", None
|
| 101 |
-
print(f"Fetched {len(questions_data)} questions.")
|
| 102 |
-
except Exception as e:
|
| 103 |
-
error_details = traceback.format_exc()
|
| 104 |
-
print(f"Error fetching questions: {e}\n{error_details}")
|
| 105 |
-
return f"Error fetching questions: {e}", None
|
| 106 |
-
|
| 107 |
-
# Run agent on questions
|
| 108 |
-
results_log = []
|
| 109 |
-
answers_payload = []
|
| 110 |
-
print(f"Running agent on {len(questions_data)} questions...")
|
| 111 |
-
|
| 112 |
-
# Progress tracking
|
| 113 |
-
total_questions = len(questions_data)
|
| 114 |
-
completed = 0
|
| 115 |
-
failed = 0
|
| 116 |
-
|
| 117 |
-
for item in questions_data:
|
| 118 |
-
task_id = item.get("task_id")
|
| 119 |
-
question_text = item.get("question")
|
| 120 |
-
if not task_id or question_text is None:
|
| 121 |
-
print(f"Skipping item with missing task_id or question: {item}")
|
| 122 |
-
continue
|
| 123 |
-
|
| 124 |
-
try:
|
| 125 |
-
# Update progress
|
| 126 |
-
completed += 1
|
| 127 |
-
print(f"Processing question {completed}/{total_questions}: Task ID {task_id}")
|
| 128 |
-
|
| 129 |
-
# Check if the question has an associated file
|
| 130 |
-
file_path = None
|
| 131 |
-
try:
|
| 132 |
-
file_content = api_client.get_file(task_id)
|
| 133 |
-
print(f"Downloaded file for task {task_id}")
|
| 134 |
-
file_path = save_task_file(file_content, task_id)
|
| 135 |
-
except Exception as file_e:
|
| 136 |
-
print(f"No file found for task {task_id} or error: {file_e}")
|
| 137 |
-
|
| 138 |
-
# Run the agent to get the answer
|
| 139 |
-
submitted_answer = agent.answer_question(question_text, file_path)
|
| 140 |
-
|
| 141 |
-
# Add to results
|
| 142 |
-
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
|
| 143 |
-
results_log.append({
|
| 144 |
-
"Task ID": task_id,
|
| 145 |
-
"Question": question_text,
|
| 146 |
-
"Submitted Answer": submitted_answer
|
| 147 |
-
})
|
| 148 |
-
except Exception as e:
|
| 149 |
-
# Update error count
|
| 150 |
-
failed += 1
|
| 151 |
-
error_details = traceback.format_exc()
|
| 152 |
-
print(f"Error running agent on task {task_id}: {e}\n{error_details}")
|
| 153 |
-
|
| 154 |
-
# Add error to results
|
| 155 |
-
error_msg = f"AGENT ERROR: {e}"
|
| 156 |
-
answers_payload.append({"task_id": task_id, "submitted_answer": error_msg})
|
| 157 |
-
results_log.append({
|
| 158 |
-
"Task ID": task_id,
|
| 159 |
-
"Question": question_text,
|
| 160 |
-
"Submitted Answer": error_msg
|
| 161 |
-
})
|
| 162 |
-
|
| 163 |
-
# Print summary
|
| 164 |
-
print(f"\nProcessing complete: {completed} questions processed, {failed} failures")
|
| 165 |
-
|
| 166 |
-
if not answers_payload:
|
| 167 |
-
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
|
| 168 |
-
|
| 169 |
-
# Submit answers
|
| 170 |
-
submission_data = {
|
| 171 |
-
"username": username.strip(),
|
| 172 |
-
"agent_code": agent_code,
|
| 173 |
-
"answers": answers_payload
|
| 174 |
-
}
|
| 175 |
-
|
| 176 |
-
print(f"Submitting {len(answers_payload)} answers for username '{username}'...")
|
| 177 |
-
|
| 178 |
-
try:
|
| 179 |
-
result_data = api_client.submit_answers(
|
| 180 |
-
username.strip(),
|
| 181 |
-
agent_code,
|
| 182 |
-
answers_payload
|
| 183 |
-
)
|
| 184 |
-
|
| 185 |
-
# Calculate success rate
|
| 186 |
-
correct_count = result_data.get('correct_count', 0)
|
| 187 |
-
total_attempted = result_data.get('total_attempted', len(answers_payload))
|
| 188 |
-
success_rate = (correct_count / total_attempted) * 100 if total_attempted > 0 else 0
|
| 189 |
-
|
| 190 |
-
final_status = (
|
| 191 |
-
f"Submission Successful!\n"
|
| 192 |
-
f"User: {result_data.get('username')}\n"
|
| 193 |
-
f"Overall Score: {result_data.get('score', 'N/A')}% "
|
| 194 |
-
f"({correct_count}/{total_attempted} correct, {success_rate:.1f}% success rate)\n"
|
| 195 |
-
f"Message: {result_data.get('message', 'No message received.')}"
|
| 196 |
-
)
|
| 197 |
-
|
| 198 |
-
print("Submission successful.")
|
| 199 |
-
return final_status, pd.DataFrame(results_log)
|
| 200 |
-
except Exception as e:
|
| 201 |
-
error_details = traceback.format_exc()
|
| 202 |
-
status_message = f"Submission Failed: {e}\n{error_details}"
|
| 203 |
-
print(status_message)
|
| 204 |
-
return status_message, pd.DataFrame(results_log)
|
| 205 |
-
|
| 206 |
-
# Build Gradio Interface
|
| 207 |
-
with gr.Blocks() as demo:
|
| 208 |
-
gr.Markdown("# GAIA Agent Evaluation Runner")
|
| 209 |
-
gr.Markdown(
|
| 210 |
-
"""
|
| 211 |
-
**Instructions:**
|
| 212 |
-
|
| 213 |
-
1. Log in to your Hugging Face account using the button below.
|
| 214 |
-
2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score.
|
| 215 |
-
|
| 216 |
-
**Configuration:**
|
| 217 |
-
|
| 218 |
-
You can configure the agent by setting these environment variables:
|
| 219 |
-
- `AGENT_MODEL_TYPE`: Model type (HfApiModel, InferenceClientModel, LiteLLMModel, OpenAIServerModel)
|
| 220 |
-
- `AGENT_MODEL_ID`: Model ID
|
| 221 |
-
- `AGENT_TEMPERATURE`: Temperature for generation (0.0-1.0)
|
| 222 |
-
- `AGENT_EXECUTOR_TYPE`: Type of executor ('local' or 'e2b')
|
| 223 |
-
- `AGENT_VERBOSE`: Enable verbose logging (true/false)
|
| 224 |
-
- `AGENT_API_BASE`: Base URL for API calls (for OpenAIServerModel)
|
| 225 |
-
|
| 226 |
-
**xAI Support:**
|
| 227 |
-
- `XAI_API_KEY`: Your xAI API key
|
| 228 |
-
- `XAI_API_BASE`: Base URL for xAI API (default: https://api.groq.com/openai/v1)
|
| 229 |
-
- When using xAI, set AGENT_MODEL_TYPE=OpenAIServerModel and AGENT_MODEL_ID=mixtral-8x7b-32768
|
| 230 |
-
|
| 231 |
-
**InferenceClientModel specific settings:**
|
| 232 |
-
- `AGENT_PROVIDER`: Provider for InferenceClientModel (e.g., "hf-inference")
|
| 233 |
-
- `AGENT_TIMEOUT`: Timeout in seconds for API calls
|
| 234 |
-
"""
|
| 235 |
-
)
|
| 236 |
-
|
| 237 |
-
gr.LoginButton()
|
| 238 |
-
|
| 239 |
-
run_button = gr.Button("Run Evaluation & Submit All Answers")
|
| 240 |
-
|
| 241 |
-
status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
|
| 242 |
-
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
|
| 243 |
-
|
| 244 |
-
run_button.click(
|
| 245 |
-
fn=run_and_submit_all,
|
| 246 |
-
outputs=[status_output, results_table]
|
| 247 |
-
)
|
| 248 |
-
|
| 249 |
-
if __name__ == "__main__":
|
| 250 |
-
print("\n" + "-"*30 + " App Starting " + "-"*30)
|
| 251 |
-
|
| 252 |
-
# Check for environment variables
|
| 253 |
-
config = get_agent_configuration()
|
| 254 |
-
print(f"Agent configuration: {config}")
|
| 255 |
-
|
| 256 |
-
# Run the Gradio app
|
| 257 |
-
demo.launch(debug=True, share=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -1,10 +1,2 @@
|
|
| 1 |
gradio
|
| 2 |
-
requests
|
| 3 |
-
smolagents[openai]
|
| 4 |
-
python-dotenv
|
| 5 |
-
pandas
|
| 6 |
-
numpy
|
| 7 |
-
openai
|
| 8 |
-
transformers
|
| 9 |
-
huggingface_hub
|
| 10 |
-
torch
|
|
|
|
| 1 |
gradio
|
| 2 |
+
requests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|