Update app.py

app.py

@@ -1,4 +1,6 @@
 import os
+import re
+import traceback
 
 import gradio as gr
 import pandas as pd
@@ -8,22 +10,27 @@ from smolagents import (
     CodeAgent,
     DuckDuckGoSearchTool,
     InferenceClientModel,
+    VisitWebpageTool,
 )
 
 
-# ---------------------------------------------------------
-# Configuration
-# ---------------------------------------------------------
+# =========================================================
+# CONFIG
+# =========================================================
 
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
+# Explicit model instead of relying on a changing default.
+MODEL_ID = "Qwen/Qwen3-Next-80B-A3B-Thinking"
 
-# ---------------------------------------------------------
-# Agent
-# ---------------------------------------------------------
+
+# =========================================================
+# AGENT
+# =========================================================
 
 class BasicAgent:
     def __init__(self):
+        print("=" * 80)
         print("Initializing GAIA agent...")
 
         hf_token = os.getenv("HF_TOKEN")
@@ -31,18 +38,28 @@ class BasicAgent:
         if not hf_token:
             raise RuntimeError(
                 "HF_TOKEN is missing. "
-                "Add it in Space -> Settings -> Variables and secrets."
+                "Add HF_TOKEN in Space -> Settings -> Variables and secrets."
             )
 
+        print(f"Using model: {MODEL_ID}")
+
         self.model = InferenceClientModel(
-            token=hf_token
+            model_id=MODEL_ID,
+            token=hf_token,
+            timeout=180,
         )
 
         self.agent = CodeAgent(
+            model=self.model,
             tools=[
-                DuckDuckGoSearchTool(),
+                DuckDuckGoSearchTool(
+                    max_results=10,
+                    rate_limit=1.0,
+                ),
+                VisitWebpageTool(
+                    max_output_length=30000,
+                ),
             ],
-            model=self.model,
             additional_authorized_imports=[
                 "requests",
                 "re",
@@ -52,284 +69,306 @@ class BasicAgent:
                 "statistics",
                 "collections",
                 "itertools",
+                "urllib.parse",
             ],
-            max_steps=12,
+            max_steps=15,
         )
 
         print("GAIA agent initialized successfully.")
+        print("=" * 80)
+
+    def clean_answer(self, value) -> str:
+        if value is None:
+            return ""
+
+        answer = str(value).strip()
+
+        # Remove common prefixes if model ignored instructions.
+        answer = re.sub(
+            r"^\s*(FINAL\s+ANSWER|Final\s+Answer|Final\s+answer)\s*:\s*",
+            "",
+            answer,
+            flags=re.IGNORECASE,
+        )
+
+        # Remove surrounding whitespace/newlines.
+        answer = answer.strip()
+
+        return answer
 
     def __call__(self, question: str) -> str:
         print("\n" + "=" * 80)
-        print("QUESTION:")
+        print("QUESTION")
+        print("=" * 80)
         print(question)
         print("=" * 80)
 
         prompt = f"""
-You are an autonomous agent solving a GAIA benchmark question.
-
-Your objective is to determine the exact final answer.
-
-You may use:
-- web search
-- Python/code
-- HTTP requests
-- calculations
-- text processing
-
-Instructions:
+You are solving one GAIA benchmark question.
 
-1. Read the question extremely carefully.
+Your goal is to obtain the exact answer, not merely explain how to
+find it.
 
-2. Determine whether the answer can be derived directly or whether
-   external research is necessary.
+You have web-search and webpage-reading tools. You can also execute
+Python code.
 
-3. When current or factual information is required, use web search.
+Work autonomously.
 
-4. When calculations, transformations, counting, decoding, reversing
-   text, parsing, or similar operations are required, use Python.
+Rules:
 
-5. Verify important facts whenever possible.
+1. Carefully determine exactly what the question asks.
 
-6. Pay very close attention to requested formatting.
+2. If the question requires factual information, search the web.
 
-7. The final response MUST contain ONLY the answer.
+3. Do not trust only a search-result snippet when the underlying page
+   can be inspected. Open relevant pages and verify the information.
 
-Do NOT include:
-- explanations
-- reasoning
-- citations
-- "FINAL ANSWER:"
-- introductory text
+4. If Wikipedia is explicitly mentioned, search for the relevant
+   Wikipedia page and inspect the required information.
 
-Examples of valid final responses:
+5. For counting, arithmetic, dates, string transformations, reversing
+   text, decoding, sorting, comparison, or other deterministic work,
+   use Python when useful.
 
-7
+6. Cross-check ambiguous information when possible.
 
-Mercedes Sosa
+7. Pay extremely close attention to the requested output format.
 
-Paris
+8. Your final answer must be minimal. Examples:
+   7
+   Paris
+   Mercedes Sosa
+   12, 15, 18
+   Qh4
 
-12, 15, 18
+9. Do NOT put explanations, citations, reasoning, Markdown, or
+   "FINAL ANSWER:" in the final answer.
 
-The answer should be as short and exact as possible.
+10. If the question contains a URL, investigate that URL or search for
+    information related to it when useful.
 
 QUESTION:
 
 {question}
+
+Solve the problem and return only the exact final answer.
 """
 
-        try:
-            result = self.agent.run(prompt)
+        result = self.agent.run(prompt)
 
-            answer = str(result).strip()
+        answer = self.clean_answer(result)
 
-            # Some models may still prepend FINAL ANSWER.
-            prefixes = [
-                "FINAL ANSWER:",
-                "Final Answer:",
-                "Final answer:",
-            ]
+        if not answer:
+            raise RuntimeError(
+                "The agent completed without returning a final answer."
+            )
 
-            for prefix in prefixes:
-                if answer.startswith(prefix):
-                    answer = answer[len(prefix):].strip()
+        print("\n" + "-" * 80)
+        print("FINAL ANSWER:")
+        print(answer)
+        print("-" * 80)
 
-            print("\nAGENT FINAL ANSWER:")
-            print(answer)
+        return answer
 
-            return answer
 
-        except Exception as e:
-            print(f"Agent error: {e}")
-            return ""
+# =========================================================
+# FETCH QUESTIONS
+# =========================================================
 
+def fetch_questions():
+    questions_url = f"{DEFAULT_API_URL}/questions"
 
-# ---------------------------------------------------------
-# Evaluation
-# ---------------------------------------------------------
+    print(f"Fetching questions from: {questions_url}")
 
-def run_and_submit_all(profile: gr.OAuthProfile | None):
+    response = requests.get(
+        questions_url,
+        timeout=30,
+    )
 
-    space_id = os.getenv("SPACE_ID")
+    response.raise_for_status()
 
-    if profile:
-        username = f"{profile.username}"
-        print(f"User logged in: {username}")
-    else:
-        print("User not logged in.")
-        return "Please Login to Hugging Face with the button.", None
+    data = response.json()
 
-    api_url = DEFAULT_API_URL
+    if not isinstance(data, list) or not data:
+        raise RuntimeError(
+            "The scoring API returned an empty or invalid questions list."
+        )
 
-    questions_url = f"{api_url}/questions"
-    submit_url = f"{api_url}/submit"
+    print(f"Fetched {len(data)} questions.")
 
-    # -----------------------------------------------------
-    # Initialize agent
-    # -----------------------------------------------------
+    return data
 
-    try:
-        agent = BasicAgent()
 
-    except Exception as e:
-        print(f"Error instantiating agent: {e}")
+# =========================================================
+# EVALUATION
+# =========================================================
+
+def run_and_submit_all(profile: gr.OAuthProfile | None):
+    if not profile:
+        return (
+            "Please log in to Hugging Face first.",
+            pd.DataFrame(),
+        )
+
+    username = str(profile.username).strip()
+
+    print("\n" + "#" * 80)
+    print(f"USER: {username}")
+    print("#" * 80)
+
+    space_id = os.getenv("SPACE_ID")
 
+    if not space_id:
         return (
-            f"Error initializing agent: {e}",
-            None,
+            "SPACE_ID environment variable is missing.",
+            pd.DataFrame(),
         )
 
-    # Public link to your agent source code
     agent_code = (
-        f"https://huggingface.co/spaces/"
-        f"{space_id}/tree/main"
+        f"https://huggingface.co/spaces/{space_id}/tree/main"
     )
 
-    print(f"Agent code: {agent_code}")
-
     # -----------------------------------------------------
-    # Fetch questions
+    # Initialize agent
     # -----------------------------------------------------
 
-    print(f"Fetching questions from: {questions_url}")
-
     try:
-        response = requests.get(
-            questions_url,
-            timeout=30,
-        )
-
-        response.raise_for_status()
-
-        questions_data = response.json()
+        agent = BasicAgent()
 
-        if not questions_data:
-            return (
-                "Fetched questions list is empty or invalid.",
-                None,
-            )
+    except Exception as e:
+        traceback.print_exc()
 
-        print(
-            f"Fetched {len(questions_data)} questions."
+        return (
+            f"Error initializing agent:\n{type(e).__name__}: {e}",
+            pd.DataFrame(),
         )
 
-    except requests.exceptions.RequestException as e:
-
-        print(f"Error fetching questions: {e}")
+    # -----------------------------------------------------
+    # Fetch questions
+    # -----------------------------------------------------
 
-        return (
-            f"Error fetching questions: {e}",
-            None,
-        )
+    try:
+        questions_data = fetch_questions()
 
     except Exception as e:
-
-        print(
-            f"Unexpected error fetching questions: {e}"
-        )
+        traceback.print_exc()
 
         return (
-            f"Unexpected error fetching questions: {e}",
-            None,
+            f"Error fetching questions:\n{type(e).__name__}: {e}",
+            pd.DataFrame(),
         )
 
     # -----------------------------------------------------
-    # Run agent
+    # Solve
     # -----------------------------------------------------
 
     results_log = []
     answers_payload = []
 
-    print(
-        f"Running agent on "
-        f"{len(questions_data)} questions..."
-    )
+    success_count = 0
+    error_count = 0
 
-    for index, item in enumerate(
-        questions_data,
-        start=1,
-    ):
+    total = len(questions_data)
 
+    for index, item in enumerate(questions_data, start=1):
         task_id = item.get("task_id")
-        question_text = item.get("question")
+        question = item.get("question")
 
-        if not task_id or question_text is None:
-            print(
-                f"Skipping invalid question: {item}"
-            )
+        if not task_id or question is None:
+            print(f"Skipping malformed task: {item}")
             continue
 
         print("\n" + "#" * 80)
+        print(f"TASK {index}/{total}")
+        print(f"TASK ID: {task_id}")
+        print("#" * 80)
 
-        print(
-            f"QUESTION "
-            f"{index}/{len(questions_data)}"
-        )
+        try:
+            answer = agent(question)
 
-        print(f"Task ID: {task_id}")
+            success_count += 1
 
-        try:
-            submitted_answer = agent(
-                question_text
+            answers_payload.append(
+                {
+                    "task_id": task_id,
+                    "submitted_answer": answer,
+                }
+            )
+
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question,
+                    "Submitted Answer": answer,
+                    "Status": "OK",
+                }
             )
 
         except Exception as e:
+            error_count += 1
 
-            print(
-                f"Error running agent on "
-                f"{task_id}: {e}"
+            traceback.print_exc()
+
+            error_message = (
+                f"{type(e).__name__}: {str(e)}"
             )
 
-            submitted_answer = ""
+            print(
+                f"\nERROR ON TASK {task_id}: "
+                f"{error_message}"
+            )
 
-        answers_payload.append(
-            {
-                "task_id": task_id,
-                "submitted_answer":
-                    submitted_answer,
-            }
-        )
+            # IMPORTANT:
+            # Do NOT send an empty answer to the scoring API.
+            results_log.append(
+                {
+                    "Task ID": task_id,
+                    "Question": question,
+                    "Submitted Answer": "",
+                    "Status": error_message,
+                }
+            )
 
-        results_log.append(
-            {
-                "Task ID": task_id,
-                "Question": question_text,
-                "Submitted Answer":
-                    submitted_answer,
-            }
-        )
+    results_df = pd.DataFrame(results_log)
 
     # -----------------------------------------------------
-    # Ensure answers exist
+    # Prevent another useless 0/20 submission
     # -----------------------------------------------------
 
-    if not answers_payload:
-
+    if success_count == 0:
         return (
-            "Agent did not produce any answers.",
-            pd.DataFrame(results_log),
+            "Evaluation stopped.\n\n"
+            f"Successful answers: 0/{total}\n"
+            f"Errors: {error_count}/{total}\n\n"
+            "Nothing was submitted because the model failed to "
+            "produce any answers.\n\n"
+            "Open Logs and inspect the Status column below.",
+            results_df,
         )
 
+    print("\n" + "=" * 80)
+    print("AGENT RUN COMPLETE")
+    print(f"Successful answers: {success_count}")
+    print(f"Errors: {error_count}")
+    print("=" * 80)
+
     # -----------------------------------------------------
-    # Submission payload
+    # Submit only successfully generated answers
     # -----------------------------------------------------
 
     submission_data = {
-        "username": username.strip(),
+        "username": username,
         "agent_code": agent_code,
         "answers": answers_payload,
     }
 
+    submit_url = f"{DEFAULT_API_URL}/submit"
+
     print(
-        f"Agent finished. Submitting "
-        f"{len(answers_payload)} answers "
-        f"for user '{username}'..."
+        f"Submitting {len(answers_payload)} answers "
+        f"to {submit_url}"
     )
 
-    # -----------------------------------------------------
-    # Submit
-    # -----------------------------------------------------
-
     try:
         response = requests.post(
             submit_url,
@@ -339,137 +378,88 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
         response.raise_for_status()
 
-        result_data = response.json()
-
-        final_status = (
-            "Submission Successful!\n"
-            f"User: "
-            f"{result_data.get('username')}\n"
-            f"Overall Score: "
-            f"{result_data.get('score', 'N/A')}% "
-            f"("
-            f"{result_data.get('correct_count', '?')}/"
-            f"{result_data.get('total_attempted', '?')}"
-            f" correct)\n"
-            f"Message: "
-            f"{result_data.get('message', 'No message received.')}"
-        )
-
-        print(final_status)
-
-        return (
-            final_status,
-            pd.DataFrame(results_log),
-        )
-
-    except requests.exceptions.HTTPError as e:
-
-        error_detail = (
-            f"Server responded with status "
-            f"{e.response.status_code}."
-        )
-
-        try:
-            error_json = e.response.json()
-
-            error_detail += (
-                f" Detail: "
-                f"{error_json.get('detail', e.response.text)}"
-            )
+        result = response.json()
 
-        except Exception:
-            error_detail += (
-                f" Response: "
-                f"{e.response.text[:500]}"
-            )
-
-        status_message = (
-            f"Submission Failed: {error_detail}"
+        score = result.get("score", "N/A")
+        correct = result.get("correct_count", "?")
+        attempted = result.get("total_attempted", "?")
+        message = result.get(
+            "message",
+            "No message received.",
         )
 
-        print(status_message)
-
-        return (
-            status_message,
-            pd.DataFrame(results_log),
+        status = (
+            "Submission Successful!\n\n"
+            f"User: {result.get('username', username)}\n"
+            f"Generated answers: {success_count}/{total}\n"
+            f"Agent errors: {error_count}/{total}\n"
+            f"Score: {score}%\n"
+            f"Correct: {correct}/{attempted}\n\n"
+            f"{message}"
         )
 
-    except requests.exceptions.Timeout:
-
-        status_message = (
-            "Submission Failed: "
-            "The request timed out."
-        )
+        print(status)
 
-        print(status_message)
+        return status, results_df
 
-        return (
-            status_message,
-            pd.DataFrame(results_log),
-        )
-
-    except requests.exceptions.RequestException as e:
-
-        status_message = (
-            f"Submission Failed: "
-            f"Network error - {e}"
-        )
+    except requests.exceptions.HTTPError as e:
+        traceback.print_exc()
 
-        print(status_message)
+        try:
+            detail = e.response.json()
+        except Exception:
+            detail = e.response.text[:1000]
 
         return (
-            status_message,
-            pd.DataFrame(results_log),
+            "Submission failed.\n\n"
+            f"HTTP {e.response.status_code}\n"
+            f"{detail}",
+            results_df,
         )
 
     except Exception as e:
-
-        status_message = (
-            f"Unexpected submission error: {e}"
-        )
-
-        print(status_message)
+        traceback.print_exc()
 
         return (
-            status_message,
-            pd.DataFrame(results_log),
+            "Submission failed.\n\n"
+            f"{type(e).__name__}: {e}",
+            results_df,
         )
 
 
-# ---------------------------------------------------------
-# Gradio UI
-# ---------------------------------------------------------
+# =========================================================
+# UI
+# =========================================================
 
 with gr.Blocks() as demo:
-
-    gr.Markdown(
-        "# GAIA Agent Evaluation Runner"
-    )
-
     gr.Markdown(
         """
+# GAIA Agent Evaluation Runner
+
 ### Instructions
 
 1. Log in with your Hugging Face account.
 2. Click **Run Evaluation & Submit All Answers**.
-3. The agent will fetch the GAIA questions.
-4. It will solve each question.
-5. Answers will automatically be submitted.
-6. Your score will appear below.
+3. The agent will fetch the course questions.
+4. It will research and solve them.
+5. Successful answers will be submitted to the course scoring API.
+6. Errors will be displayed in the table instead of silently
+   submitting blank answers.
 
-The evaluation can take several minutes.
+Evaluation may take several minutes.
 """
     )
 
     gr.LoginButton()
 
     run_button = gr.Button(
-        "Run Evaluation & Submit All Answers"
+        "Run Evaluation & Submit All Answers",
+        variant="primary",
     )
 
     status_output = gr.Textbox(
         label="Run Status / Submission Result",
-        lines=6,
+        lines=10,
         interactive=False,
     )
 
@@ -487,26 +477,17 @@ The evaluation can take several minutes.
     )
 
 
-# ---------------------------------------------------------
-# Start
-# ---------------------------------------------------------
+# =========================================================
+# START
+# =========================================================
 
 if __name__ == "__main__":
-
     print("\n" + "=" * 80)
-
-    print("Starting GAIA Agent Space")
-
-    print(
-        f"SPACE_ID: "
-        f"{os.getenv('SPACE_ID', 'not found')}"
-    )
-
-    print(
-        f"SPACE_HOST: "
-        f"{os.getenv('SPACE_HOST', 'not found')}"
-    )
-
+    print("STARTING GAIA AGENT SPACE")
+    print(f"SPACE_ID: {os.getenv('SPACE_ID', 'NOT FOUND')}")
+    print(f"SPACE_HOST: {os.getenv('SPACE_HOST', 'NOT FOUND')}")
+    print(f"HF_TOKEN configured: {bool(os.getenv('HF_TOKEN'))}")
+    print(f"MODEL: {MODEL_ID}")
     print("=" * 80 + "\n")
 
     demo.launch(