File size: 4,962 Bytes
1872480
10e9b7d
 
eccf8e4
3c4371f
10e9b7d
1872480
 
 
 
3db6293
e80aab9
1872480
 
 
 
31243f4
1872480
31243f4
1872480
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31243f4
1872480
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31243f4
 
 
1872480
 
 
eccf8e4
1872480
 
 
 
7d65c66
1872480
 
7d65c66
1872480
 
 
 
 
 
 
 
 
 
 
31243f4
1872480
 
 
31243f4
1872480
 
 
31243f4
1872480
 
 
 
 
 
 
 
 
 
 
 
 
31243f4
1872480
31243f4
1872480
 
31243f4
1872480
 
 
 
 
 
e80aab9
 
1872480
 
 
 
 
 
e80aab9
1872480
 
 
 
 
 
 
 
 
e80aab9
1872480
 
 
7d65c66
1872480
 
e80aab9
1872480
 
 
e80aab9
 
1872480
 
 
0ee0419
1872480
e80aab9
 
7e4a06b
e80aab9
1872480
 
 
 
 
 
 
 
e80aab9
1872480
 
 
 
e80aab9
31243f4
 
 
e80aab9
 
1872480
e80aab9
1872480
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208

import os
import gradio as gr
import requests
import pandas as pd

from smolagents import CodeAgent, InferenceClientModel
from tools import DuckDuckGoSearchTool, WeatherInfoTool, HubStatsTool
from retriever import load_guest_dataset

DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"

# -------------------------------
# Create Alfred AI Agent
# -------------------------------

class BasicAgent:

    def __init__(self):
        token = os.getenv("HF_TOKEN")

        if not token:
            raise ValueError(
                "HF_TOKEN is missing. Add it in Space Secrets."
            )

        model = InferenceClientModel(
            model_id="Qwen/Qwen2.5-Coder-32B-Instruct",
            token=token
        )

        self.search_tool = DuckDuckGoSearchTool()
        self.weather_tool = WeatherInfoTool()
        self.hub_tool = HubStatsTool()
        self.guest_tool = load_guest_dataset()

        self.agent = CodeAgent(
            tools=[
                self.guest_tool,
                self.weather_tool,
                self.hub_tool,
                self.search_tool
            ],
            model=model,
            max_steps=8,
            planning_interval=3
        )

    def __call__(self, question: str) -> str:
        try:
            answer = self.agent.run(
                question,
                reset=True
            )

            if answer is None:
                return "No answer generated."

            return str(answer).strip()

        except Exception as e:
            print(f"Agent error: {e}")
            return f"Agent execution error: {e}"


# -------------------------------
# Evaluation and Submission
# -------------------------------

def run_and_submit_all(profile: gr.OAuthProfile | None):

    if not profile:
        return "Please login to Hugging Face first.", None

    username = profile.username
    space_id = os.getenv("SPACE_ID")

    if not space_id:
        return "SPACE_ID not found. Run this inside your Hugging Face Space.", None

    agent_code = (
        f"https://huggingface.co/spaces/{space_id}/tree/main"
    )

    # Initialize agent
    try:
        agent = BasicAgent()
    except Exception as e:
        return f"Agent initialization failed: {e}", None

    # Fetch GAIA questions
    try:
        response = requests.get(
            f"{DEFAULT_API_URL}/questions",
            timeout=30
        )
        response.raise_for_status()
        questions = response.json()

    except Exception as e:
        return f"Could not fetch questions: {e}", None

    if not questions:
        return "No questions received.", None

    answers = []
    results = []

    # Solve questions
    for index, item in enumerate(questions, start=1):

        task_id = item.get("task_id")
        question = item.get("question")

        if not task_id or not question:
            continue

        print(f"Solving question {index}/{len(questions)}")

        try:
            answer = agent(question)

            answers.append({
                "task_id": task_id,
                "submitted_answer": answer
            })

            results.append({
                "Task ID": task_id,
                "Question": question,
                "Submitted Answer": answer
            })

        except Exception as e:
            print(f"Question error: {e}")

    if not answers:
        return "Agent could not generate any answers.", pd.DataFrame(results)

    # Submit answers
    submission_data = {
        "username": username,
        "agent_code": agent_code,
        "answers": answers
    }

    try:
        response = requests.post(
            f"{DEFAULT_API_URL}/submit",
            json=submission_data,
            timeout=300
        )

        response.raise_for_status()
        result = response.json()

        status = (
            "Submission Successful!\n"
            f"User: {result.get('username')}\n"
            f"Score: {result.get('score')}%\n"
            f"Correct: {result.get('correct_count')}/"
            f"{result.get('total_attempted')}\n"
            f"Message: {result.get('message')}"
        )

        return status, pd.DataFrame(results)

    except Exception as e:
        return f"Submission failed: {e}", pd.DataFrame(results)


# -------------------------------
# Gradio Interface
# -------------------------------

with gr.Blocks() as demo:

    gr.Markdown("# Alfred - GAIA AI Agent")

    gr.Markdown(
        "Login to Hugging Face and run your AI agent evaluation."
    )

    gr.LoginButton()

    run_button = gr.Button(
        "Run Evaluation & Submit All Answers"
    )

    status_output = gr.Textbox(
        label="Submission Result",
        lines=6
    )

    results_table = gr.DataFrame(
        label="Agent Answers",
        wrap=True
    )

    run_button.click(
        fn=run_and_submit_all,
        outputs=[status_output, results_table]
    )


if __name__ == "__main__":
    demo.launch()