Joachim M. Vis commited on
Commit
47bf6c8
·
verified ·
1 Parent(s): a529285

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +353 -28
app.py CHANGED
@@ -17,7 +17,12 @@ import gradio as gr
17
  import logging
18
  import time
19
  import traceback
20
- from typing import Dict, Any, Optional
 
 
 
 
 
21
 
22
  # Import our GAIA Agent
23
  from gaia_agent import GAIAAgent
@@ -324,6 +329,319 @@ def clear_conversation(agent_instance: Optional[GAIAAgent] = None) -> tuple:
324
  else:
325
  return [], "Conversation history cleared. Note: Agent memory may not have been cleared."
326
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
327
  def create_interface():
328
  """
329
  Create and configure the Gradio interface
@@ -356,34 +674,41 @@ def create_interface():
356
  else:
357
  gr.Markdown("✅ **Agent initialized successfully.**")
358
 
359
- # Chat interface
360
- with gr.Group():
361
- chatbot = gr.Chatbot(
362
- label="Conversation",
363
- elem_id="gaia_chatbot",
364
- height=450,
365
- show_copy_button=True
366
- )
367
-
368
- with gr.Row():
369
- msg = gr.Textbox(
370
- label="Your question",
371
- placeholder="Ask me anything...",
372
- lines=2,
373
- max_lines=10,
374
- show_label=False,
375
- container=False
376
- )
377
- submit_btn = gr.Button("Submit", variant="primary")
 
 
 
 
 
 
 
 
 
 
 
 
 
378
 
379
- with gr.Row():
380
- clear_btn = gr.Button("Clear Conversation")
381
- error_box = gr.Textbox(
382
- label="Status",
383
- visible=True,
384
- interactive=False,
385
- show_label=False
386
- )
387
 
388
  # About section
389
  with gr.Accordion("About GAIA Agent", open=False):
 
17
  import logging
18
  import time
19
  import traceback
20
+ import requests
21
+ import pandas as pd
22
+ from typing import Dict, Any, Optional, Tuple, List
23
+
24
+ # Constants
25
+ DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
26
 
27
  # Import our GAIA Agent
28
  from gaia_agent import GAIAAgent
 
329
  else:
330
  return [], "Conversation history cleared. Note: Agent memory may not have been cleared."
331
 
332
+ def run_and_submit_all(
333
+ profile: Optional[gr.OAuthProfile] = None,
334
+ submission_username: str = "",
335
+ agent_code: str = "",
336
+ agent_instance: Optional[GAIAAgent] = None,
337
+ progress=gr.Progress()
338
+ ) -> Tuple[str, Optional[pd.DataFrame]]:
339
+ """
340
+ Fetches all questions, runs the GAIA Agent on them, submits all answers,
341
+ and displays the results.
342
+
343
+ Args:
344
+ profile: User OAuth profile (if using OAuth)
345
+ submission_username: Username for submission (if not using OAuth)
346
+ agent_code: URL to the agent code repository
347
+ agent_instance: GAIA Agent instance
348
+ progress: Gradio progress tracker
349
+
350
+ Returns:
351
+ Tuple of (status_message, results_dataframe)
352
+ """
353
+ # Configure logging
354
+ logger = logging.getLogger("gaia_evaluation")
355
+
356
+ # --- Determine HF Space Runtime URL and Repo URL ---
357
+ space_id = os.getenv("SPACE_ID", "")
358
+
359
+ # Set username based on profile or input
360
+ if profile:
361
+ username = profile.username
362
+ logger.info(f"User logged in: {username}")
363
+ elif submission_username:
364
+ username = submission_username
365
+ logger.info(f"Using provided username: {username}")
366
+ else:
367
+ logger.warning("No username provided for submission")
368
+ return "Please enter a username or log in to Hugging Face.", None
369
+
370
+ # Set API endpoints
371
+ api_url = DEFAULT_API_URL
372
+ questions_url = f"{api_url}/questions"
373
+ submit_url = f"{api_url}/submit"
374
+
375
+ # Set agent code URL if not provided
376
+ if not agent_code and space_id:
377
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
378
+ logger.info(f"Using default agent code URL: {agent_code}")
379
+
380
+ # Check if agent is available
381
+ if agent_instance is None:
382
+ error_msg = "GAIA Agent is not initialized. Please check the application logs."
383
+ logger.error(error_msg)
384
+ return error_msg, None
385
+
386
+ # Initialize progress
387
+ progress(0, desc="Initializing evaluation...")
388
+
389
+ # Fetch questions
390
+ progress(0.15, desc="Fetching questions...")
391
+ logger.info(f"Fetching questions from: {questions_url}")
392
+ try:
393
+ response = requests.get(questions_url, timeout=15)
394
+ response.raise_for_status()
395
+ questions_data = response.json()
396
+ if not questions_data:
397
+ error_msg = "Fetched questions list is empty or invalid format."
398
+ logger.warning(error_msg)
399
+ return error_msg, None
400
+
401
+ question_count = len(questions_data)
402
+ logger.info(f"Fetched {question_count} questions.")
403
+ progress(0.2, desc=f"Fetched {question_count} questions")
404
+ except requests.exceptions.RequestException as e:
405
+ error_msg = f"Error fetching questions: {e}"
406
+ logger.error(error_msg)
407
+ return error_msg, None
408
+ except requests.exceptions.JSONDecodeError as e:
409
+ error_msg = f"Error decoding JSON response from questions endpoint: {e}"
410
+ logger.error(f"{error_msg}\nResponse text: {response.text[:500]}")
411
+ return error_msg, None
412
+ except Exception as e:
413
+ error_msg = f"An unexpected error occurred fetching questions: {e}"
414
+ logger.error(error_msg)
415
+ logger.error(traceback.format_exc())
416
+ return error_msg, None
417
+
418
+ # Process questions with the agent
419
+ results_log = []
420
+ answers_payload = []
421
+ total_questions = len(questions_data)
422
+
423
+ logger.info(f"Running agent on {total_questions} questions...")
424
+
425
+ question_progress_increment = 0.6 / total_questions # 60% of progress bar for processing questions
426
+
427
+ for i, item in enumerate(questions_data):
428
+ # Extract question information - handle both formats from different API versions
429
+ task_id = item.get("task_id", item.get("id", f"q{i+1}"))
430
+ question_text = item.get("question")
431
+
432
+ if not task_id or question_text is None:
433
+ logger.warning(f"Skipping item with missing task_id or question: {item}")
434
+ continue
435
+
436
+ progress_value = 0.2 + (i * question_progress_increment)
437
+ progress(progress_value, desc=f"Processing question {i+1}/{total_questions}...")
438
+
439
+ logger.info(f"Processing question {i+1}/{total_questions}: {question_text[:50]}...")
440
+ question_start_time = time.time()
441
+
442
+ try:
443
+ # Call the appropriate agent method
444
+ submitted_answer = agent_instance.process_question(question_text)
445
+
446
+ processing_time = time.time() - question_start_time
447
+
448
+ # Format for submit payload
449
+ answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
450
+
451
+ # Store results for display
452
+ results_log.append({
453
+ "Task ID": task_id,
454
+ "Question": question_text,
455
+ "Submitted Answer": submitted_answer,
456
+ "Processing Time": f"{processing_time:.2f}s",
457
+ "Status": "Success"
458
+ })
459
+
460
+ logger.info(f"Question {i+1} processed in {processing_time:.2f}s. Answer: {submitted_answer[:50]}...")
461
+
462
+ except Exception as e:
463
+ error_msg = f"Error running agent on task {task_id}: {e}"
464
+ logger.error(error_msg)
465
+ logger.error(traceback.format_exc())
466
+
467
+ results_log.append({
468
+ "Task ID": task_id,
469
+ "Question": question_text,
470
+ "Submitted Answer": f"AGENT ERROR: {e}",
471
+ "Processing Time": f"{time.time() - question_start_time:.2f}s",
472
+ "Status": "Error"
473
+ })
474
+
475
+ # Check if we have answers to submit
476
+ if not answers_payload:
477
+ error_msg = "Agent did not produce any answers to submit."
478
+ logger.warning(error_msg)
479
+ return error_msg, pd.DataFrame(results_log) if results_log else None
480
+
481
+ # Prepare submission
482
+ progress(0.8, desc="Preparing submission...")
483
+
484
+ # Format submission payload
485
+ submission_data = {
486
+ "username": username.strip(),
487
+ "agent_code": agent_code,
488
+ "answers": answers_payload
489
+ }
490
+
491
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
492
+ logger.info(status_update)
493
+
494
+ # Submit answers
495
+ progress(0.9, desc=f"Submitting answers...")
496
+ logger.info(f"Submitting answers to: {submit_url}")
497
+
498
+ try:
499
+ response = requests.post(submit_url, json=submission_data, timeout=60)
500
+ response.raise_for_status()
501
+ result_data = response.json()
502
+
503
+ # Extract results
504
+ correct_count = result_data.get('correct_count', 0)
505
+ total_attempted = result_data.get('total_attempted', 0)
506
+ score = result_data.get('score', 'N/A')
507
+
508
+ final_status = (
509
+ f"[SUCCESS] Submission Successful!\n"
510
+ f"User: {result_data.get('username', username)}\n"
511
+ f"Overall Score: {score}% "
512
+ f"({correct_count}/{total_attempted} correct)\n"
513
+ f"Message: {result_data.get('message', 'No message received.')}"
514
+ )
515
+ logger.info("Submission successful.")
516
+
517
+ # Create results DataFrame
518
+ results_df = pd.DataFrame(results_log)
519
+
520
+ # Calculate statistics if we have results
521
+ if not results_df.empty:
522
+ success_count = sum(1 for status in results_df.get("Status", []) if status == "Success")
523
+ error_count = sum(1 for status in results_df.get("Status", []) if status == "Error")
524
+
525
+ try:
526
+ avg_time = sum(float(time.strip('s')) for time in results_df["Processing Time"]) / len(results_df)
527
+ avg_time_str = f"{avg_time:.2f}s"
528
+ except:
529
+ avg_time_str = "N/A"
530
+
531
+ logger.info(f"Processed {len(results_df)} questions: {success_count} successes, {error_count} errors, avg time: {avg_time_str}")
532
+
533
+ # Complete progress and return results
534
+ progress(1.0, desc="Evaluation complete!")
535
+ return final_status, results_df
536
+
537
+ except requests.exceptions.HTTPError as e:
538
+ error_detail = f"Server responded with status {e.response.status_code}."
539
+ try:
540
+ error_json = e.response.json()
541
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
542
+ except:
543
+ error_detail += f" Response: {e.response.text[:500]}"
544
+ status_message = f"Submission Failed: {error_detail}"
545
+ logger.error(status_message)
546
+ results_df = pd.DataFrame(results_log) if results_log else None
547
+ return status_message, results_df
548
+
549
+ except requests.exceptions.Timeout:
550
+ status_message = "Submission Failed: The request timed out."
551
+ logger.error(status_message)
552
+ results_df = pd.DataFrame(results_log) if results_log else None
553
+ return status_message, results_df
554
+
555
+ except requests.exceptions.RequestException as e:
556
+ status_message = f"Submission Failed: Network error - {e}"
557
+ logger.error(status_message)
558
+ results_df = pd.DataFrame(results_log) if results_log else None
559
+ return status_message, results_df
560
+
561
+ except Exception as e:
562
+ status_message = f"An unexpected error occurred during submission: {e}"
563
+ logger.error(status_message)
564
+ logger.error(traceback.format_exc())
565
+ results_df = pd.DataFrame(results_log) if results_log else None
566
+ return status_message, results_df
567
+
568
+ def create_evaluation_tab(demo, agent_instance):
569
+ """
570
+ Creates and configures the evaluation tab for the Gradio interface.
571
+
572
+ Args:
573
+ demo: The Gradio Blocks instance to add the tab to
574
+ agent_instance: The GAIA Agent instance to use for evaluation
575
+
576
+ Returns:
577
+ None - modifies the demo in place
578
+ """
579
+ with gr.Tab("Run Evaluation"):
580
+ gr.Markdown("## GAIA Agent Evaluation")
581
+
582
+ with gr.Row():
583
+ with gr.Column(scale=2):
584
+ gr.Markdown(
585
+ """
586
+ This tab allows you to evaluate the GAIA Agent against a set of benchmark questions.
587
+
588
+ The evaluation process:
589
+ 1. Initializes the GAIA Agent
590
+ 2. Fetches questions from the evaluation server
591
+ 3. Runs the agent on each question
592
+ 4. Submits the answers and retrieves scores
593
+ 5. Displays detailed results
594
+
595
+ You can either log in with your Hugging Face account or enter a username for the submission.
596
+ """
597
+ )
598
+
599
+ with gr.Column(scale=1):
600
+ # OAuth login option
601
+ gr.LoginButton(visible=True)
602
+
603
+ # Alternative username field
604
+ username_input = gr.Textbox(
605
+ label="Username for Submission",
606
+ placeholder="Enter your username here if not using HF login",
607
+ lines=1
608
+ )
609
+
610
+ # Optional code URL field
611
+ code_url_input = gr.Textbox(
612
+ label="Agent Code URL (optional)",
613
+ placeholder="URL to your code repository",
614
+ lines=1,
615
+ value=""
616
+ )
617
+
618
+ # Run button
619
+ run_button = gr.Button("Run Evaluation & Submit All Answers", variant="primary")
620
+
621
+ # Output components
622
+ status_output = gr.Textbox(
623
+ label="Evaluation Status / Submission Result",
624
+ lines=5,
625
+ interactive=False
626
+ )
627
+
628
+ results_table = gr.DataFrame(
629
+ label="Questions and Agent Answers",
630
+ wrap=True,
631
+ visible=True
632
+ )
633
+
634
+ # Set up event handler
635
+ run_button.click(
636
+ fn=run_and_submit_all,
637
+ inputs=[
638
+ username_input,
639
+ code_url_input,
640
+ agent_instance
641
+ ],
642
+ outputs=[status_output, results_table]
643
+ )
644
+
645
  def create_interface():
646
  """
647
  Create and configure the Gradio interface
 
674
  else:
675
  gr.Markdown("✅ **Agent initialized successfully.**")
676
 
677
+ # Create tabs for chat and evaluation
678
+ with gr.Tabs():
679
+ # Chat tab
680
+ with gr.Tab("Chat"):
681
+ # Chat interface
682
+ with gr.Group():
683
+ chatbot = gr.Chatbot(
684
+ label="Conversation",
685
+ elem_id="gaia_chatbot",
686
+ height=450,
687
+ show_copy_button=True
688
+ )
689
+
690
+ with gr.Row():
691
+ msg = gr.Textbox(
692
+ label="Your question",
693
+ placeholder="Ask me anything...",
694
+ lines=2,
695
+ max_lines=10,
696
+ show_label=False,
697
+ container=False
698
+ )
699
+ submit_btn = gr.Button("Submit", variant="primary")
700
+
701
+ with gr.Row():
702
+ clear_btn = gr.Button("Clear Conversation")
703
+ error_box = gr.Textbox(
704
+ label="Status",
705
+ visible=True,
706
+ interactive=False,
707
+ show_label=False
708
+ )
709
 
710
+ # Add evaluation tab
711
+ create_evaluation_tab(demo, agent_instance)
 
 
 
 
 
 
712
 
713
  # About section
714
  with gr.Accordion("About GAIA Agent", open=False):