app.py

import os
import gradio as gr
import requests
import pandas as pd

from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel

# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"


# --- Agent Definition ---
class BasicAgent:
    def __init__(self):
        print("Initializing agent...")

        self.model = HfApiModel(
            model_id="Qwen/Qwen2.5-Coder-32B-Instruct"
        )

        self.agent = CodeAgent(
            tools=[
                DuckDuckGoSearchTool()
            ],
            model=self.model,
            max_steps=10
        )

        print("Agent initialized successfully.")

    def __call__(self, question: str) -> str:
        print(f"Agent received question: {question[:150]}...")

        prompt = f"""
You are an expert research agent answering a benchmark question.

Solve the question carefully and accurately.

IMPORTANT INSTRUCTIONS:

1. Analyze exactly what the question is asking.
2. Use web search whenever external information is needed.
3. Prefer reliable sources and cross-check important facts.
4. If the question refers to Wikipedia, search for the relevant
   Wikipedia information.
5. If the question contains a URL, investigate the URL when possible.
6. If the question requires calculations, use Python/code when useful.
7. If the question asks for a specific format, follow that format exactly.
8. Do not guess when you can research the answer.
9. Return ONLY the final answer.
10. Do not include your reasoning or analysis.
11. Do not say "Here is the answer".
12. Do not add unnecessary explanation.

Question:
{question}
"""

        try:
            result = self.agent.run(prompt)

            answer = str(result).strip()

            print(f"Agent answer: {answer}")

            return answer

        except Exception as e:
            print(f"Agent error: {e}")
            return f"Unable to answer: {e}"


# --- Evaluation and Submission ---
def run_and_submit_all(profile: gr.OAuthProfile | None):
    """
    Fetch all benchmark questions, run the agent,
    submit the answers, and display the results.
    """

    space_id = os.getenv("SPACE_ID")

    if profile:
        username = f"{profile.username}"
        print(f"User logged in: {username}")
    else:
        return "Please Login to Hugging Face with the button.", None

    api_url = DEFAULT_API_URL
    questions_url = f"{api_url}/questions"
    submit_url = f"{api_url}/submit"

    # --- Initialize Agent ---
    try:
        agent = BasicAgent()
    except Exception as e:
        print(f"Error initializing agent: {e}")
        return f"Error initializing agent: {e}", None

    # --- Link to the Space code ---
    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"

    # --- Fetch Questions ---
    print(f"Fetching questions from: {questions_url}")

    try:
        response = requests.get(
            questions_url,
            timeout=30
        )

        response.raise_for_status()

        questions_data = response.json()

        if not questions_data:
            return "No questions were returned.", None

        print(f"Fetched {len(questions_data)} questions.")

    except requests.exceptions.RequestException as e:
        return f"Error fetching questions: {e}", None

    except Exception as e:
        return f"Unexpected error fetching questions: {e}", None

    # --- Run Agent on Questions ---
    results_log = []
    answers_payload = []

    print(f"Running agent on {len(questions_data)} questions...")

    for index, item in enumerate(questions_data, start=1):

        task_id = item.get("task_id")
        question_text = item.get("question")

        if not task_id or question_text is None:
            print(f"Skipping invalid item: {item}")
            continue

        print("=" * 70)
        print(f"Question {index}/{len(questions_data)}")
        print(f"Task ID: {task_id}")

        try:
            submitted_answer = agent(question_text)

            answers_payload.append({
                "task_id": task_id,
                "submitted_answer": submitted_answer
            })

            results_log.append({
                "Task ID": task_id,
                "Question": question_text,
                "Submitted Answer": submitted_answer
            })

            print(f"Submitted answer: {submitted_answer}")

        except Exception as e:

            print(f"Agent error on task {task_id}: {e}")

            results_log.append({
                "Task ID": task_id,
                "Question": question_text,
                "Submitted Answer": f"AGENT ERROR: {e}"
            })

    # --- Make Sure We Have Answers ---
    if not answers_payload:
        return (
            "Agent did not produce any answers.",
            pd.DataFrame(results_log)
        )

    # --- Prepare Submission ---
    submission_data = {
        "username": username.strip(),
        "agent_code": agent_code,
        "answers": answers_payload
    }

    print(
        f"Submitting {len(answers_payload)} answers "
        f"for user {username}..."
    )

    # --- Submit Answers ---
    try:

        response = requests.post(
            submit_url,
            json=submission_data,
            timeout=120
        )

        response.raise_for_status()

        result_data = response.json()

        final_status = (
            f"Submission Successful!\n"
            f"User: {result_data.get('username')}\n"
            f"Overall Score: "
            f"{result_data.get('score', 'N/A')}% "
            f"({result_data.get('correct_count', '?')}/"
            f"{result_data.get('total_attempted', '?')} correct)\n"
            f"Message: "
            f"{result_data.get('message', 'No message received.')}"
        )

        print(final_status)

        return (
            final_status,
            pd.DataFrame(results_log)
        )

    except requests.exceptions.HTTPError as e:

        error_detail = (
            f"Server responded with status "
            f"{e.response.status_code}."
        )

        try:
            error_json = e.response.json()

            error_detail += (
                f" Detail: "
                f"{error_json.get('detail', e.response.text)}"
            )

        except Exception:
            error_detail += (
                f" Response: {e.response.text[:500]}"
            )

        return (
            f"Submission Failed: {error_detail}",
            pd.DataFrame(results_log)
        )

    except requests.exceptions.Timeout:

        return (
            "Submission Failed: The request timed out.",
            pd.DataFrame(results_log)
        )

    except requests.exceptions.RequestException as e:

        return (
            f"Submission Failed: Network error - {e}",
            pd.DataFrame(results_log)
        )

    except Exception as e:

        return (
            f"Unexpected submission error: {e}",
            pd.DataFrame(results_log)
        )


# --- Gradio Interface ---
with gr.Blocks() as demo:

    gr.Markdown("# Basic Agent Evaluation Runner")

    gr.Markdown(
        """
        **Instructions:**

        1. Log in to Hugging Face.
        2. Click **Run Evaluation & Submit All Answers**.
        3. The agent will fetch the benchmark questions.
        4. The agent will use web search to research questions.
        5. Answers will be submitted automatically.

        **Note:** Evaluation can take several minutes.
        """
    )

    gr.LoginButton()

    run_button = gr.Button(
        "Run Evaluation & Submit All Answers"
    )

    status_output = gr.Textbox(
        label="Run Status / Submission Result",
        lines=5,
        interactive=False
    )

    results_table = gr.DataFrame(
        label="Questions and Agent Answers",
        wrap=True
    )

    run_button.click(
        fn=run_and_submit_all,
        outputs=[
            status_output,
            results_table
        ]
    )


# --- Start Application ---
if __name__ == "__main__":

    print("\n" + "-" * 30 + " App Starting " + "-" * 30)

    space_host = os.getenv("SPACE_HOST")
    space_id = os.getenv("SPACE_ID")

    if space_host:
        print(f"SPACE_HOST: {space_host}")

    if space_id:
        print(f"SPACE_ID: {space_id}")
        print(
            f"Repository: "
            f"https://huggingface.co/spaces/{space_id}"
        )

    print("-" * 70)

    demo.launch(
        debug=True,
        share=False
    )