jm-vis commited on
Commit
0c64865
·
1 Parent(s): 47bf6c8

Fix UI and evaluation functionality with simplified app.py

Browse files
Files changed (1) hide show
  1. app.py +133 -725
app.py CHANGED
@@ -1,30 +1,17 @@
1
- """
2
- GAIA Agent Hugging Face Space Interface
3
-
4
- A clean, straightforward interface for interacting with the GAIA Agent.
5
- This implementation focuses on providing a smooth user experience with
6
- proper error handling and clear messaging.
7
-
8
- Environment variables:
9
- - OPENAI_API_KEY: Required for the LLM
10
- - SERPER_API_KEY: For web search capabilities
11
- - SUPABASE_URL: Optional for memory storage
12
- - SUPABASE_KEY: Optional for memory storage
13
- """
14
-
15
  import os
16
  import gradio as gr
 
 
17
  import logging
18
  import time
19
  import traceback
20
- import requests
21
- import pandas as pd
22
  from typing import Dict, Any, Optional, Tuple, List
23
 
24
- # Constants
 
25
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
26
 
27
- # Import our GAIA Agent
28
  from gaia_agent import GAIAAgent
29
 
30
  # Configure logging
@@ -32,423 +19,104 @@ logging.basicConfig(
32
  level=logging.INFO,
33
  format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
34
  )
35
- logger = logging.getLogger("gaia_app")
36
-
37
- # Load environment variables with error handling
38
- def get_env_var(name, default="", required=False):
39
- """Helper function to get environment variables with logging"""
40
- value = os.getenv(name, default)
41
- if not value and required:
42
- logger.error(f"Required environment variable {name} is not set")
43
- elif not value:
44
- logger.warning(f"Optional environment variable {name} is not set")
45
- return value
46
-
47
- # Check for required environment variables with default values
48
- OPENAI_API_KEY = get_env_var("OPENAI_API_KEY", "", True) # Critical for agent operation
49
- SERPER_API_KEY = get_env_var("SERPER_API_KEY", "") # Optional but recommended
50
- PERPLEXITY_API_KEY = get_env_var("PERPLEXITY_API_KEY", "") # Optional alternative to SERPER
51
- SUPABASE_URL = get_env_var("SUPABASE_URL", "") # Optional for memory persistence
52
- SUPABASE_KEY = get_env_var("SUPABASE_KEY", "") # Optional for memory persistence
53
- SPACE_HOST = get_env_var("SPACE_HOST", "") # For Hugging Face Space detection
54
- SPACE_ID = get_env_var("SPACE_ID", "") # For Hugging Face Space identification
55
-
56
- # Validate critical dependencies
57
- try:
58
- import gradio as gr
59
- except ImportError as e:
60
- logger.critical(f"Critical dependency 'gradio' not found: {e}")
61
- raise ImportError("Gradio is required for the app interface. Please install it with 'pip install gradio'.")
62
-
63
- # Log warnings and detailed diagnostics for missing environment variables
64
- if not OPENAI_API_KEY:
65
- logger.warning("OPENAI_API_KEY is not set. Agent will use fallback mode with limited capabilities.")
66
- logger.info("To get an OpenAI API key, visit: https://platform.openai.com/api-keys")
67
-
68
- # Check search capabilities
69
- search_capability = "full"
70
- if not SERPER_API_KEY and not PERPLEXITY_API_KEY:
71
- search_capability = "limited"
72
- logger.warning("Neither SERPER_API_KEY nor PERPLEXITY_API_KEY is set. Web search capabilities will be limited to DuckDuckGo.")
73
- logger.info("To get a Serper API key, visit: https://serper.dev/")
74
- elif not SERPER_API_KEY:
75
- search_capability = "partial"
76
- logger.warning("SERPER_API_KEY is not set. Will attempt to use Perplexity for search if available.")
77
- elif not PERPLEXITY_API_KEY:
78
- search_capability = "partial"
79
- logger.warning("PERPLEXITY_API_KEY is not set. Will use Serper for search capabilities.")
80
 
81
- # Check memory persistence
82
- if not SUPABASE_URL or not SUPABASE_KEY:
83
- logger.warning("SUPABASE_URL or SUPABASE_KEY is not set. Memory persistence will be limited to in-memory storage.")
84
- if SUPABASE_URL and not SUPABASE_KEY:
85
- logger.error("SUPABASE_URL is set but SUPABASE_KEY is missing. Memory persistence will not work correctly.")
86
- elif SUPABASE_KEY and not SUPABASE_URL:
87
- logger.error("SUPABASE_KEY is set but SUPABASE_URL is missing. Memory persistence will not work correctly.")
88
-
89
- # Log overall capability assessment
90
- logger.info(f"GAIA Agent capabilities: LLM: {'available' if OPENAI_API_KEY else 'unavailable'}, "
91
- f"Search: {search_capability}, "
92
- f"Memory: {'persistent' if (SUPABASE_URL and SUPABASE_KEY) else 'in-memory only'}")
93
-
94
- # Custom CSS for improved appearance
95
- custom_css = """
96
- .message-bubble {
97
- padding: 10px 15px;
98
- border-radius: 15px;
99
- margin-bottom: 10px;
100
- }
101
- .user-message {
102
- background-color: #e6f7ff;
103
- border: 1px solid #91d5ff;
104
- }
105
- .agent-message {
106
- background-color: #f6ffed;
107
- border: 1px solid #b7eb8f;
108
- }
109
- .banner {
110
- padding: 10px;
111
- background-color: #f9f9f9;
112
- border-radius: 8px;
113
- margin-bottom: 15px;
114
- border: 1px solid #e0e0e0;
115
- }
116
- """
117
-
118
- def initialize_agent() -> Optional[GAIAAgent]:
119
- """
120
- Initialize the GAIA Agent with proper error handling
121
-
122
- Returns:
123
- GAIAAgent instance or None if initialization fails
124
- """
125
  try:
126
- start_time = time.time()
127
  agent = GAIAAgent()
128
- init_time = time.time() - start_time
129
- logger.info(f"GAIA Agent initialized successfully in {init_time:.2f} seconds")
130
  return agent
131
  except Exception as e:
132
- error_msg = f"Error initializing GAIA agent: {e}"
133
- logger.error(error_msg)
134
  logger.error(traceback.format_exc())
135
  return None
136
 
137
- def check_environment() -> Dict[str, bool]:
138
- """
139
- Check if all required environment variables are set
140
-
141
- Returns:
142
- Dictionary with status of each required environment variable
143
- """
144
- return {
145
- "OPENAI_API_KEY": bool(OPENAI_API_KEY),
146
- "SERPER_API_KEY": bool(SERPER_API_KEY),
147
- "PERPLEXITY_API_KEY": bool(PERPLEXITY_API_KEY),
148
- "SUPABASE_URL": bool(SUPABASE_URL),
149
- "SUPABASE_KEY": bool(SUPABASE_KEY),
150
- }
151
-
152
- def process_question(
153
- message: str,
154
- history: list,
155
- agent_instance: Optional[GAIAAgent] = None,
156
- progress=gr.Progress()
157
- ) -> tuple:
158
- """
159
- Process a user question and get a response from the GAIA Agent
160
-
161
- Args:
162
- message: User's question
163
- history: Conversation history
164
- agent_instance: GAIA Agent instance
165
- progress: Gradio progress indicator
166
-
167
- Returns:
168
- Tuple of (history, error_message)
169
- """
170
- logger.info(f"Processing question request with message length: {len(message) if message else 0}")
171
-
172
- # Input validation with detailed error messages
173
- if not message:
174
- logger.warning("Empty question received")
175
- return history, "Please enter a question."
176
-
177
- if not isinstance(message, str):
178
- logger.error(f"Invalid message type: {type(message)}. Expected string.")
179
- return history, "Invalid input type. Please enter a text question."
180
-
181
- if not message.strip():
182
- logger.warning("Blank question (only whitespace) received")
183
- return history, "Please enter a non-empty question."
184
-
185
- # Validate history parameter
186
- if not isinstance(history, list):
187
- logger.error(f"Invalid history type: {type(history)}. Expected list.")
188
- history = [] # Reset to empty list for recovery
189
-
190
- # Check if agent is available with enhanced diagnostics
191
- if agent_instance is None:
192
- logger.error("Agent instance is None when processing question")
193
- # Generate a more specific error message based on environment variables
194
- env_status = check_environment()
195
- missing_vars = [key for key, value in env_status.items() if not value]
196
-
197
- # Check if OPENAI_API_KEY is missing, as it's critical
198
- critical_vars_missing = "OPENAI_API_KEY" in missing_vars
199
-
200
- if missing_vars:
201
- if critical_vars_missing:
202
- error_detail = f"Critical environment variables missing: {', '.join(missing_vars)}"
203
- logger.error(error_detail)
204
- else:
205
- error_detail = f"Some environment variables are missing: {', '.join(missing_vars)}"
206
- logger.warning(error_detail)
207
- else:
208
- error_detail = "Unknown initialization error. Please check the application logs."
209
- logger.error("Agent is None but no environment variables are missing")
210
-
211
- user_message = "I'm unable to process your question because the AI agent isn't properly initialized."
212
- if critical_vars_missing:
213
- user_message += " This is likely due to missing API keys required for my operation."
214
-
215
- return (
216
- history + [(message, user_message)],
217
- f"GAIA Agent initialization failed. {error_detail}"
218
- )
219
-
220
- # Initialize progress indicator with better error handling
221
- try:
222
- progress(0.1, desc="Processing question...")
223
- except Exception as progress_err:
224
- logger.warning(f"Non-critical error with progress indicator: {progress_err}")
225
- # Continue execution - progress indicator issues shouldn't block processing
226
-
227
- error_message = ""
228
-
229
- try:
230
- # Process the question with timing
231
- start_time = time.time()
232
- logger.info(f"Sending question to agent: '{message[:50]}{'...' if len(message) > 50 else ''}'")
233
-
234
- try:
235
- progress(0.4, desc="Thinking...")
236
- except:
237
- pass # Non-critical, continue even if progress update fails
238
-
239
- response = agent_instance.process_question(message)
240
-
241
- try:
242
- progress(1.0, desc="Done!")
243
- except:
244
- pass # Non-critical
245
-
246
- processing_time = time.time() - start_time
247
- logger.info(f"Question processed successfully in {processing_time:.2f} seconds")
248
-
249
- # Validate response
250
- if not response or not isinstance(response, str):
251
- logger.warning(f"Agent returned invalid response type: {type(response)}")
252
- response = "I generated a response but it was in an invalid format. Please try asking again."
253
-
254
- # Return the updated history and empty error message
255
- return history + [(message, response)], ""
256
-
257
- except Exception as e:
258
- error_details = traceback.format_exc()
259
- error_type = type(e).__name__
260
- logger.error(f"Error processing question: {error_type}: {e}")
261
- logger.error(error_details)
262
-
263
- # Create user-friendly error message based on exception type
264
- user_message = "I encountered an error while processing your question."
265
-
266
- if "ConnectionError" in error_type or "Timeout" in error_type:
267
- user_message += " It seems I'm having trouble connecting to my knowledge services. This might be a temporary network issue."
268
- elif "JSONDecodeError" in error_type:
269
- user_message += " I had trouble processing some data. This might be due to an unexpected format."
270
- elif "AuthenticationError" in error_type or "api key" in str(e).lower():
271
- user_message += " There seems to be an issue with my authentication to external services."
272
- elif "MemoryError" in error_type or "OutOfMemoryError" in error_type:
273
- user_message += " The question might be too complex for me to process with current resources."
274
- else:
275
- user_message += " Please try again or rephrase your question."
276
-
277
- error_message = f"Error processing question: {error_type}: {str(e)}"
278
- return history + [(message, user_message)], error_message
279
-
280
- def clear_conversation(agent_instance: Optional[GAIAAgent] = None) -> tuple:
281
- """
282
- Clear the conversation history and agent memory
283
-
284
- Args:
285
- agent_instance: GAIA Agent instance
286
-
287
- Returns:
288
- Empty history and status message
289
- """
290
- logger.info("Clear conversation request received")
291
-
292
- # Track completion status to provide feedback
293
- agent_memory_cleared = False
294
-
295
- if agent_instance:
296
- try:
297
- # Validate agent instance has clear_memory method
298
- if not hasattr(agent_instance, 'clear_memory') or not callable(getattr(agent_instance, 'clear_memory')):
299
- logger.error("Agent instance does not have clear_memory method or it's not callable")
300
- return [], "Failed to clear conversation: Invalid agent configuration."
301
-
302
- # Clear the agent memory with detailed logging
303
- start_time = time.time()
304
- agent_instance.clear_memory()
305
- clear_time = time.time() - start_time
306
-
307
- logger.info(f"Agent memory cleared successfully in {clear_time:.2f} seconds")
308
- agent_memory_cleared = True
309
-
310
- except Exception as e:
311
- error_type = type(e).__name__
312
- error_details = traceback.format_exc()
313
- logger.error(f"Error clearing agent memory: {error_type}: {e}")
314
- logger.error(error_details)
315
-
316
- # Return more specific error message based on exception type
317
- if "AttributeError" in error_type:
318
- return [], "Failed to clear conversation: Agent memory interface error."
319
- elif "ConnectionError" in error_type:
320
- return [], "Failed to clear persistent memory: Connection issue."
321
- else:
322
- return [], f"Partially cleared conversation: {error_type}."
323
- else:
324
- logger.warning("Clear conversation called with None agent instance")
325
-
326
- # Provide different message based on what was cleared
327
- if agent_memory_cleared:
328
- return [], "Conversation and agent memory cleared successfully."
329
- else:
330
- return [], "Conversation history cleared. Note: Agent memory may not have been cleared."
331
-
332
- def run_and_submit_all(
333
- profile: Optional[gr.OAuthProfile] = None,
334
- submission_username: str = "",
335
- agent_code: str = "",
336
- agent_instance: Optional[GAIAAgent] = None,
337
- progress=gr.Progress()
338
- ) -> Tuple[str, Optional[pd.DataFrame]]:
339
  """
340
  Fetches all questions, runs the GAIA Agent on them, submits all answers,
341
  and displays the results.
342
-
343
- Args:
344
- profile: User OAuth profile (if using OAuth)
345
- submission_username: Username for submission (if not using OAuth)
346
- agent_code: URL to the agent code repository
347
- agent_instance: GAIA Agent instance
348
- progress: Gradio progress tracker
349
-
350
- Returns:
351
- Tuple of (status_message, results_dataframe)
352
  """
353
- # Configure logging
354
- logger = logging.getLogger("gaia_evaluation")
355
-
356
  # --- Determine HF Space Runtime URL and Repo URL ---
357
  space_id = os.getenv("SPACE_ID", "")
358
-
359
- # Set username based on profile or input
360
  if profile:
361
  username = profile.username
362
  logger.info(f"User logged in: {username}")
363
- elif submission_username:
364
- username = submission_username
365
- logger.info(f"Using provided username: {username}")
366
  else:
367
- logger.warning("No username provided for submission")
368
- return "Please enter a username or log in to Hugging Face.", None
369
-
370
- # Set API endpoints
371
  api_url = DEFAULT_API_URL
372
  questions_url = f"{api_url}/questions"
373
  submit_url = f"{api_url}/submit"
374
-
375
- # Set agent code URL if not provided
376
- if not agent_code and space_id:
377
- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
378
- logger.info(f"Using default agent code URL: {agent_code}")
379
-
380
- # Check if agent is available
381
- if agent_instance is None:
382
- error_msg = "GAIA Agent is not initialized. Please check the application logs."
 
383
  logger.error(error_msg)
384
  return error_msg, None
385
-
386
- # Initialize progress
387
- progress(0, desc="Initializing evaluation...")
388
-
389
- # Fetch questions
390
- progress(0.15, desc="Fetching questions...")
391
  logger.info(f"Fetching questions from: {questions_url}")
392
  try:
393
  response = requests.get(questions_url, timeout=15)
394
  response.raise_for_status()
395
  questions_data = response.json()
396
  if not questions_data:
397
- error_msg = "Fetched questions list is empty or invalid format."
398
- logger.warning(error_msg)
399
- return error_msg, None
400
-
401
- question_count = len(questions_data)
402
- logger.info(f"Fetched {question_count} questions.")
403
- progress(0.2, desc=f"Fetched {question_count} questions")
404
  except requests.exceptions.RequestException as e:
405
  error_msg = f"Error fetching questions: {e}"
406
  logger.error(error_msg)
407
  return error_msg, None
408
  except requests.exceptions.JSONDecodeError as e:
409
- error_msg = f"Error decoding JSON response from questions endpoint: {e}"
410
- logger.error(f"{error_msg}\nResponse text: {response.text[:500]}")
411
- return error_msg, None
412
  except Exception as e:
413
  error_msg = f"An unexpected error occurred fetching questions: {e}"
414
  logger.error(error_msg)
415
  logger.error(traceback.format_exc())
416
  return error_msg, None
417
-
418
- # Process questions with the agent
419
  results_log = []
420
  answers_payload = []
421
- total_questions = len(questions_data)
422
-
423
- logger.info(f"Running agent on {total_questions} questions...")
424
-
425
- question_progress_increment = 0.6 / total_questions # 60% of progress bar for processing questions
426
 
427
  for i, item in enumerate(questions_data):
428
- # Extract question information - handle both formats from different API versions
429
  task_id = item.get("task_id", item.get("id", f"q{i+1}"))
430
  question_text = item.get("question")
431
 
432
  if not task_id or question_text is None:
433
  logger.warning(f"Skipping item with missing task_id or question: {item}")
434
  continue
435
-
436
- progress_value = 0.2 + (i * question_progress_increment)
437
- progress(progress_value, desc=f"Processing question {i+1}/{total_questions}...")
438
 
439
- logger.info(f"Processing question {i+1}/{total_questions}: {question_text[:50]}...")
440
- question_start_time = time.time()
441
 
442
  try:
443
- # Call the appropriate agent method
444
- submitted_answer = agent_instance.process_question(question_text)
445
-
446
- processing_time = time.time() - question_start_time
447
 
448
- # Format for submit payload
449
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
450
-
451
- # Store results for display
452
  results_log.append({
453
  "Task ID": task_id,
454
  "Question": question_text,
@@ -457,7 +125,7 @@ def run_and_submit_all(
457
  "Status": "Success"
458
  })
459
 
460
- logger.info(f"Question {i+1} processed in {processing_time:.2f}s. Answer: {submitted_answer[:50]}...")
461
 
462
  except Exception as e:
463
  error_msg = f"Error running agent on task {task_id}: {e}"
@@ -468,72 +136,46 @@ def run_and_submit_all(
468
  "Task ID": task_id,
469
  "Question": question_text,
470
  "Submitted Answer": f"AGENT ERROR: {e}",
471
- "Processing Time": f"{time.time() - question_start_time:.2f}s",
472
  "Status": "Error"
473
  })
474
-
475
  # Check if we have answers to submit
476
  if not answers_payload:
477
- error_msg = "Agent did not produce any answers to submit."
478
- logger.warning(error_msg)
479
- return error_msg, pd.DataFrame(results_log) if results_log else None
480
-
481
- # Prepare submission
482
- progress(0.8, desc="Preparing submission...")
483
-
484
- # Format submission payload
485
  submission_data = {
486
- "username": username.strip(),
487
- "agent_code": agent_code,
488
  "answers": answers_payload
489
  }
490
-
491
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
492
  logger.info(status_update)
493
-
494
- # Submit answers
495
- progress(0.9, desc=f"Submitting answers...")
496
- logger.info(f"Submitting answers to: {submit_url}")
497
-
498
  try:
499
  response = requests.post(submit_url, json=submission_data, timeout=60)
500
  response.raise_for_status()
501
  result_data = response.json()
502
 
503
- # Extract results
504
- correct_count = result_data.get('correct_count', 0)
505
- total_attempted = result_data.get('total_attempted', 0)
506
- score = result_data.get('score', 'N/A')
507
 
508
  final_status = (
509
- f"[SUCCESS] Submission Successful!\n"
510
  f"User: {result_data.get('username', username)}\n"
511
  f"Overall Score: {score}% "
512
  f"({correct_count}/{total_attempted} correct)\n"
513
  f"Message: {result_data.get('message', 'No message received.')}"
514
  )
515
  logger.info("Submission successful.")
516
-
517
- # Create results DataFrame
518
  results_df = pd.DataFrame(results_log)
519
-
520
- # Calculate statistics if we have results
521
- if not results_df.empty:
522
- success_count = sum(1 for status in results_df.get("Status", []) if status == "Success")
523
- error_count = sum(1 for status in results_df.get("Status", []) if status == "Error")
524
-
525
- try:
526
- avg_time = sum(float(time.strip('s')) for time in results_df["Processing Time"]) / len(results_df)
527
- avg_time_str = f"{avg_time:.2f}s"
528
- except:
529
- avg_time_str = "N/A"
530
-
531
- logger.info(f"Processed {len(results_df)} questions: {success_count} successes, {error_count} errors, avg time: {avg_time_str}")
532
-
533
- # Complete progress and return results
534
- progress(1.0, desc="Evaluation complete!")
535
  return final_status, results_df
536
-
537
  except requests.exceptions.HTTPError as e:
538
  error_detail = f"Server responded with status {e.response.status_code}."
539
  try:
@@ -543,322 +185,88 @@ def run_and_submit_all(
543
  error_detail += f" Response: {e.response.text[:500]}"
544
  status_message = f"Submission Failed: {error_detail}"
545
  logger.error(status_message)
546
- results_df = pd.DataFrame(results_log) if results_log else None
547
  return status_message, results_df
548
-
549
  except requests.exceptions.Timeout:
550
  status_message = "Submission Failed: The request timed out."
551
  logger.error(status_message)
552
- results_df = pd.DataFrame(results_log) if results_log else None
553
  return status_message, results_df
554
-
555
  except requests.exceptions.RequestException as e:
556
  status_message = f"Submission Failed: Network error - {e}"
557
  logger.error(status_message)
558
- results_df = pd.DataFrame(results_log) if results_log else None
559
  return status_message, results_df
560
-
561
  except Exception as e:
562
  status_message = f"An unexpected error occurred during submission: {e}"
563
  logger.error(status_message)
564
  logger.error(traceback.format_exc())
565
- results_df = pd.DataFrame(results_log) if results_log else None
566
  return status_message, results_df
567
 
568
- def create_evaluation_tab(demo, agent_instance):
569
- """
570
- Creates and configures the evaluation tab for the Gradio interface.
571
-
572
- Args:
573
- demo: The Gradio Blocks instance to add the tab to
574
- agent_instance: The GAIA Agent instance to use for evaluation
575
-
576
- Returns:
577
- None - modifies the demo in place
578
- """
579
- with gr.Tab("Run Evaluation"):
580
- gr.Markdown("## GAIA Agent Evaluation")
581
-
582
- with gr.Row():
583
- with gr.Column(scale=2):
584
- gr.Markdown(
585
- """
586
- This tab allows you to evaluate the GAIA Agent against a set of benchmark questions.
587
-
588
- The evaluation process:
589
- 1. Initializes the GAIA Agent
590
- 2. Fetches questions from the evaluation server
591
- 3. Runs the agent on each question
592
- 4. Submits the answers and retrieves scores
593
- 5. Displays detailed results
594
-
595
- You can either log in with your Hugging Face account or enter a username for the submission.
596
- """
597
- )
598
-
599
- with gr.Column(scale=1):
600
- # OAuth login option
601
- gr.LoginButton(visible=True)
602
-
603
- # Alternative username field
604
- username_input = gr.Textbox(
605
- label="Username for Submission",
606
- placeholder="Enter your username here if not using HF login",
607
- lines=1
608
- )
609
-
610
- # Optional code URL field
611
- code_url_input = gr.Textbox(
612
- label="Agent Code URL (optional)",
613
- placeholder="URL to your code repository",
614
- lines=1,
615
- value=""
616
- )
617
-
618
- # Run button
619
- run_button = gr.Button("Run Evaluation & Submit All Answers", variant="primary")
620
-
621
- # Output components
622
- status_output = gr.Textbox(
623
- label="Evaluation Status / Submission Result",
624
- lines=5,
625
- interactive=False
626
- )
627
-
628
- results_table = gr.DataFrame(
629
- label="Questions and Agent Answers",
630
- wrap=True,
631
- visible=True
632
- )
633
-
634
- # Set up event handler
635
- run_button.click(
636
- fn=run_and_submit_all,
637
- inputs=[
638
- username_input,
639
- code_url_input,
640
- agent_instance
641
- ],
642
- outputs=[status_output, results_table]
643
- )
644
 
645
- def create_interface():
646
- """
647
- Create and configure the Gradio interface
648
-
649
- Returns:
650
- Configured Gradio Blocks interface
651
- """
652
- # Initialize agent (will be shared across sessions)
653
- agent_instance = initialize_agent()
654
- env_status = check_environment()
655
-
656
- # Construct environment status message
657
- missing_vars = [key for key, value in env_status.items() if not value]
658
- if missing_vars:
659
- env_message = f"⚠️ Missing environment variables: {', '.join(missing_vars)}"
660
- else:
661
- env_message = "✅ All environment variables are set"
662
-
663
- # Create the Gradio interface
664
- demo = gr.Blocks(css=custom_css)
665
-
666
- with demo:
667
- gr.Markdown("# GAIA Agent")
668
-
669
- # Environment status banner
670
- with gr.Box(elem_classes=["banner"]):
671
- gr.Markdown(f"### Environment Status\n{env_message}")
672
- if agent_instance is None:
673
- gr.Markdown("❌ **Agent initialization failed.** Some features may not work correctly.")
674
- else:
675
- gr.Markdown("✅ **Agent initialized successfully.**")
676
-
677
- # Create tabs for chat and evaluation
678
- with gr.Tabs():
679
- # Chat tab
680
- with gr.Tab("Chat"):
681
- # Chat interface
682
- with gr.Group():
683
- chatbot = gr.Chatbot(
684
- label="Conversation",
685
- elem_id="gaia_chatbot",
686
- height=450,
687
- show_copy_button=True
688
- )
689
-
690
- with gr.Row():
691
- msg = gr.Textbox(
692
- label="Your question",
693
- placeholder="Ask me anything...",
694
- lines=2,
695
- max_lines=10,
696
- show_label=False,
697
- container=False
698
- )
699
- submit_btn = gr.Button("Submit", variant="primary")
700
-
701
- with gr.Row():
702
- clear_btn = gr.Button("Clear Conversation")
703
- error_box = gr.Textbox(
704
- label="Status",
705
- visible=True,
706
- interactive=False,
707
- show_label=False
708
- )
709
-
710
- # Add evaluation tab
711
- create_evaluation_tab(demo, agent_instance)
712
-
713
- # About section
714
- with gr.Accordion("About GAIA Agent", open=False):
715
- gr.Markdown(
716
- """
717
- ## About GAIA Agent
718
-
719
- The GAIA (Grounded AI Assistant) Agent is designed to provide helpful and accurate responses by
720
- leveraging web search tools and robust reasoning capabilities.
721
-
722
- ### Key Features:
723
-
724
- - **Web Search:** Retrieves real-time information from the web
725
- - **Content Extraction:** Can extract and analyze content from web pages
726
- - **Structured Approach:** Uses a systematic process for analyzing questions and formulating answers
727
- - **Error Handling:** Includes robust error handling and fallback mechanisms
728
-
729
- ### Resources:
730
-
731
- - The agent is built with LangChain and LangGraph
732
- - Uses OpenAI models for reasoning and processing
733
-
734
- ### Attribution:
735
-
736
- GAIA Agent is developed as part of the Hugging Face Agents course.
737
- """
738
- )
739
-
740
- # Set up event handlers with error handling wrapper
741
- def safe_handler(fn):
742
- """Wrapper to provide additional error handling for event handlers"""
743
- def wrapped_fn(*args, **kwargs):
744
- try:
745
- return fn(*args, **kwargs)
746
- except Exception as e:
747
- error_type = type(e).__name__
748
- logger.error(f"Unhandled error in UI event handler: {error_type}: {str(e)}")
749
- logger.error(traceback.format_exc())
750
-
751
- # Determine what function was called for better error messages
752
- if fn.__name__ == "process_question":
753
- # For process_question failures, return original history + error message
754
- history = args[1] if len(args) > 1 and isinstance(args[1], list) else []
755
- user_msg = args[0] if args and isinstance(args[0], str) else "Unknown question"
756
- return history + [(user_msg, f"The system encountered an unexpected error. Please try again later.")], f"UI handler error: {error_type}"
757
- else:
758
- # For other function failures (e.g., clear_conversation)
759
- return [], f"Operation failed due to a system error: {error_type}"
760
- # Preserve original function metadata
761
- wrapped_fn.__name__ = fn.__name__
762
- wrapped_fn.__doc__ = fn.__doc__
763
- return wrapped_fn
764
-
765
- # Use the safe handler wrapper for all event handlers
766
- try:
767
- # Set up submit button handler
768
- submit_btn.click(
769
- fn=safe_handler(process_question),
770
- inputs=[msg, chatbot, agent_instance],
771
- outputs=[chatbot, error_box],
772
- api_name="submit"
773
- )
774
- logger.info("Submit button handler registered successfully")
775
-
776
- # Set up message input submission handler
777
- msg.submit(
778
- fn=safe_handler(process_question),
779
- inputs=[msg, chatbot, agent_instance],
780
- outputs=[chatbot, error_box],
781
- api_name="ask"
782
- )
783
- logger.info("Message submission handler registered successfully")
784
-
785
- # Set up clear button handler
786
- clear_btn.click(
787
- fn=safe_handler(clear_conversation),
788
- inputs=[agent_instance],
789
- outputs=[chatbot, error_box],
790
- api_name="clear"
791
- )
792
- logger.info("Clear button handler registered successfully")
793
- except Exception as handler_err:
794
- logger.error(f"Failed to register event handlers: {str(handler_err)}")
795
- logger.error(traceback.format_exc())
796
- # Add error message to the interface
797
- error_box.value = "Warning: Some interface features may not be working correctly due to initialization errors."
798
-
799
- return demo
800
 
801
  if __name__ == "__main__":
802
- try:
803
- # Check if running in a Hugging Face Space
804
- space_host = os.getenv("SPACE_HOST", "")
805
- space_id = os.getenv("SPACE_ID", "")
806
- running_in_space = bool(space_host and space_id)
807
-
808
- if running_in_space:
809
- logger.info(f"Running on Hugging Face Space: {space_host}")
810
- logger.info(f"Space ID: {space_id}")
811
- logger.info("Debug mode disabled for production environment")
812
- debug_mode = False
813
- else:
814
- logger.info("Running in local development environment")
815
- debug_mode = True
816
-
817
- # Create and launch the interface
818
- start_time = time.time()
819
- demo = create_interface()
820
- launch_time = time.time() - start_time
821
- logger.info(f"Interface created in {launch_time:.2f} seconds")
822
-
823
- # Configure launch parameters based on environment
824
- if running_in_space:
825
- # Production settings for Hugging Face Space
826
- demo.launch(
827
- debug=False,
828
- share=False,
829
- show_error=True,
830
- server_name="0.0.0.0"
831
- )
832
- else:
833
- # Development settings for local testing
834
- demo.launch(
835
- debug=True,
836
- share=False,
837
- show_error=True
838
- )
839
-
840
- except Exception as e:
841
- error_type = type(e).__name__
842
- logger.critical(f"Fatal error launching application: {error_type}: {str(e)}")
843
- logger.critical(traceback.format_exc())
844
-
845
- # Create a minimal emergency interface to show the error
846
- try:
847
- with gr.Blocks(title="GAIA Agent - Error") as emergency_ui:
848
- gr.Markdown("# ❌ GAIA Agent - Critical Error")
849
- gr.Markdown(f"**Error Type**: {error_type}")
850
- gr.Markdown(f"**Error Message**: {str(e)}")
851
- gr.Markdown("Please check the application logs for details.")
852
- gr.Markdown("### Common Solutions:")
853
- gr.Markdown("- Check that required environment variables are set")
854
- gr.Markdown("- Verify network connectivity for API access")
855
- gr.Markdown("- Ensure all dependencies are correctly installed")
856
-
857
- emergency_ui.launch(debug=True, share=False)
858
- except:
859
- # If even the emergency UI fails, print to console as last resort
860
- print("=" * 80)
861
- print("CRITICAL ERROR: Could not launch GAIA Agent application")
862
- print(f"Error: {error_type}: {str(e)}")
863
- print("Check log files for details.")
864
- print("=" * 80)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import os
2
  import gradio as gr
3
+ import requests
4
+ import pandas as pd
5
  import logging
6
  import time
7
  import traceback
 
 
8
  from typing import Dict, Any, Optional, Tuple, List
9
 
10
+ # (Keep Constants as is)
11
+ # --- Constants ---
12
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
13
 
14
+ # --- Import our GAIA Agent ---
15
  from gaia_agent import GAIAAgent
16
 
17
  # Configure logging
 
19
  level=logging.INFO,
20
  format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
21
  )
22
+ logger = logging.getLogger("gaia_evaluation")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
 
24
+ # Initialize the GAIA Agent
25
+ def initialize_agent():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  try:
 
27
  agent = GAIAAgent()
28
+ logger.info("GAIA Agent initialized successfully")
 
29
  return agent
30
  except Exception as e:
31
+ logger.error(f"Error initializing GAIA Agent: {e}")
 
32
  logger.error(traceback.format_exc())
33
  return None
34
 
35
+ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
  """
37
  Fetches all questions, runs the GAIA Agent on them, submits all answers,
38
  and displays the results.
 
 
 
 
 
 
 
 
 
 
39
  """
 
 
 
40
  # --- Determine HF Space Runtime URL and Repo URL ---
41
  space_id = os.getenv("SPACE_ID", "")
42
+
 
43
  if profile:
44
  username = profile.username
45
  logger.info(f"User logged in: {username}")
 
 
 
46
  else:
47
+ logger.warning("User not logged in.")
48
+ return "Please Login to Hugging Face with the button.", None
49
+
 
50
  api_url = DEFAULT_API_URL
51
  questions_url = f"{api_url}/questions"
52
  submit_url = f"{api_url}/submit"
53
+
54
+ # 1. Instantiate Agent
55
+ try:
56
+ agent = initialize_agent()
57
+ if agent is None:
58
+ error_msg = "Error initializing GAIA Agent. Check logs for details."
59
+ logger.error(error_msg)
60
+ return error_msg, None
61
+ except Exception as e:
62
+ error_msg = f"Error instantiating agent: {e}"
63
  logger.error(error_msg)
64
  return error_msg, None
65
+
66
+ # Link to the code repository in Hugging Face Space
67
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
68
+ logger.info(f"Using agent code URL: {agent_code}")
69
+
70
+ # 2. Fetch Questions
71
  logger.info(f"Fetching questions from: {questions_url}")
72
  try:
73
  response = requests.get(questions_url, timeout=15)
74
  response.raise_for_status()
75
  questions_data = response.json()
76
  if not questions_data:
77
+ logger.warning("Fetched questions list is empty.")
78
+ return "Fetched questions list is empty or invalid format.", None
79
+ logger.info(f"Fetched {len(questions_data)} questions.")
 
 
 
 
80
  except requests.exceptions.RequestException as e:
81
  error_msg = f"Error fetching questions: {e}"
82
  logger.error(error_msg)
83
  return error_msg, None
84
  except requests.exceptions.JSONDecodeError as e:
85
+ error_msg = f"Error decoding JSON response from questions endpoint: {e}"
86
+ logger.error(f"{error_msg}\nResponse text: {response.text[:500]}")
87
+ return error_msg, None
88
  except Exception as e:
89
  error_msg = f"An unexpected error occurred fetching questions: {e}"
90
  logger.error(error_msg)
91
  logger.error(traceback.format_exc())
92
  return error_msg, None
93
+
94
+ # 3. Run the GAIA Agent on all questions
95
  results_log = []
96
  answers_payload = []
97
+ logger.info(f"Running agent on {len(questions_data)} questions...")
 
 
 
 
98
 
99
  for i, item in enumerate(questions_data):
100
+ # Extract question information
101
  task_id = item.get("task_id", item.get("id", f"q{i+1}"))
102
  question_text = item.get("question")
103
 
104
  if not task_id or question_text is None:
105
  logger.warning(f"Skipping item with missing task_id or question: {item}")
106
  continue
 
 
 
107
 
108
+ logger.info(f"Processing question {i+1}/{len(questions_data)}: {question_text[:50]}...")
109
+ start_time = time.time()
110
 
111
  try:
112
+ # Process the question with the GAIA Agent
113
+ submitted_answer = agent.process_question(question_text)
114
+ processing_time = time.time() - start_time
 
115
 
116
+ # Add to submission payload
117
  answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
118
+
119
+ # Store for results display
120
  results_log.append({
121
  "Task ID": task_id,
122
  "Question": question_text,
 
125
  "Status": "Success"
126
  })
127
 
128
+ logger.info(f"Question {i+1} processed successfully in {processing_time:.2f}s")
129
 
130
  except Exception as e:
131
  error_msg = f"Error running agent on task {task_id}: {e}"
 
136
  "Task ID": task_id,
137
  "Question": question_text,
138
  "Submitted Answer": f"AGENT ERROR: {e}",
139
+ "Processing Time": f"{time.time() - start_time:.2f}s",
140
  "Status": "Error"
141
  })
142
+
143
  # Check if we have answers to submit
144
  if not answers_payload:
145
+ logger.warning("Agent did not produce any answers to submit.")
146
+ return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
147
+
148
+ # 4. Prepare Submission
 
 
 
 
149
  submission_data = {
150
+ "username": username.strip(),
151
+ "agent_code": agent_code,
152
  "answers": answers_payload
153
  }
 
154
  status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
155
  logger.info(status_update)
156
+
157
+ # 5. Submit
158
+ logger.info(f"Submitting {len(answers_payload)} answers to: {submit_url}")
 
 
159
  try:
160
  response = requests.post(submit_url, json=submission_data, timeout=60)
161
  response.raise_for_status()
162
  result_data = response.json()
163
 
164
+ # Extract results data
165
+ correct_count = result_data.get("correct_count", 0)
166
+ total_attempted = result_data.get("total_attempted", 0)
167
+ score = result_data.get("score", "N/A")
168
 
169
  final_status = (
170
+ f"Submission Successful!\n"
171
  f"User: {result_data.get('username', username)}\n"
172
  f"Overall Score: {score}% "
173
  f"({correct_count}/{total_attempted} correct)\n"
174
  f"Message: {result_data.get('message', 'No message received.')}"
175
  )
176
  logger.info("Submission successful.")
 
 
177
  results_df = pd.DataFrame(results_log)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
178
  return final_status, results_df
 
179
  except requests.exceptions.HTTPError as e:
180
  error_detail = f"Server responded with status {e.response.status_code}."
181
  try:
 
185
  error_detail += f" Response: {e.response.text[:500]}"
186
  status_message = f"Submission Failed: {error_detail}"
187
  logger.error(status_message)
188
+ results_df = pd.DataFrame(results_log)
189
  return status_message, results_df
 
190
  except requests.exceptions.Timeout:
191
  status_message = "Submission Failed: The request timed out."
192
  logger.error(status_message)
193
+ results_df = pd.DataFrame(results_log)
194
  return status_message, results_df
 
195
  except requests.exceptions.RequestException as e:
196
  status_message = f"Submission Failed: Network error - {e}"
197
  logger.error(status_message)
198
+ results_df = pd.DataFrame(results_log)
199
  return status_message, results_df
 
200
  except Exception as e:
201
  status_message = f"An unexpected error occurred during submission: {e}"
202
  logger.error(status_message)
203
  logger.error(traceback.format_exc())
204
+ results_df = pd.DataFrame(results_log)
205
  return status_message, results_df
206
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
207
 
208
+ # --- Build Gradio Interface using Blocks ---
209
+ with gr.Blocks() as demo:
210
+ gr.Markdown("# GAIA Agent Evaluation Runner")
211
+ gr.Markdown(
212
+ """
213
+ **Instructions:**
214
+
215
+ 1. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
216
+ 2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run the GAIA agent, submit answers, and see the score.
217
+
218
+ ---
219
+ **Note:** Running the evaluation may take some time as the agent processes all questions. Please be patient.
220
+ """
221
+ )
222
+
223
+ gr.LoginButton()
224
+
225
+ run_button = gr.Button("Run Evaluation & Submit All Answers")
226
+
227
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
228
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
229
+
230
+ run_button.click(
231
+ fn=run_and_submit_all,
232
+ outputs=[status_output, results_table]
233
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
234
 
235
  if __name__ == "__main__":
236
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
237
+ # Check for SPACE_HOST and SPACE_ID at startup for information
238
+ space_host_startup = os.getenv("SPACE_HOST")
239
+ space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup
240
+
241
+ if space_host_startup:
242
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
243
+ print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
244
+ else:
245
+ print("ℹ️ SPACE_HOST environment variable not found (running locally?).")
246
+
247
+ if space_id_startup: # Print repo URLs if SPACE_ID is found
248
+ print(f"✅ SPACE_ID found: {space_id_startup}")
249
+ print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
250
+ print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
251
+ else:
252
+ print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")
253
+
254
+ print("-"*(60 + len(" App Starting ")) + "\n")
255
+
256
+ print("Launching Gradio Interface for GAIA Agent Evaluation...")
257
+ # Determine launch parameters based on environment
258
+ is_running_in_space = bool(space_host_startup and space_id_startup)
259
+
260
+ if is_running_in_space:
261
+ # Production settings for Hugging Face Space
262
+ demo.launch(
263
+ debug=False,
264
+ share=False,
265
+ server_name="0.0.0.0"
266
+ )
267
+ else:
268
+ # Development settings for local testing
269
+ demo.launch(
270
+ debug=True,
271
+ share=False
272
+ )