File size: 8,158 Bytes
f6f7a30
10e9b7d
0e8fcd1
2a0e88c
 
0e8fcd1
 
 
 
 
 
4386f51
0e8fcd1
 
 
 
 
 
 
 
 
 
 
 
4365068
0e8fcd1
 
 
 
 
 
 
 
 
 
4365068
2a0e88c
0e8fcd1
 
 
 
 
 
 
 
 
4365068
0e8fcd1
 
4365068
 
0e8fcd1
4365068
0e8fcd1
 
 
 
4365068
0e8fcd1
 
 
 
4365068
 
0e8fcd1
4365068
 
 
 
2a0e88c
0e8fcd1
 
 
 
 
3de0945
4365068
 
 
0e8fcd1
 
 
 
4386f51
28bf034
 
0e8fcd1
 
 
4365068
0e8fcd1
 
 
 
 
 
3de0945
7436a5a
 
 
 
0e8fcd1
 
 
 
 
3de0945
0e8fcd1
 
 
 
 
 
 
4386f51
0e8fcd1
 
3de0945
 
4365068
0e8fcd1
3de0945
0e8fcd1
3de0945
 
0e8fcd1
3de0945
4365068
0e8fcd1
4365068
0e8fcd1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4365068
3c4371f
28bf034
0e8fcd1
 
 
 
 
 
3de0945
 
 
 
e80aab9
31243f4
0e8fcd1
31243f4
3de0945
 
0e8fcd1
eccf8e4
3de0945
 
0e8fcd1
7d65c66
3de0945
28bf034
0e8fcd1
 
 
 
4386f51
0e8fcd1
 
31243f4
f6f7a30
 
31243f4
4386f51
0e8fcd1
 
f6f7a30
0e8fcd1
f6f7a30
0e8fcd1
f6f7a30
0e8fcd1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f6f7a30
 
0e8fcd1
 
f6f7a30
e80aab9
4386f51
 
 
28bf034
e80aab9
4386f51
0e8fcd1
 
e80aab9
7d65c66
28bf034
e80aab9
0e8fcd1
 
e80aab9
0e8fcd1
a5b7567
 
0e8fcd1
 
 
 
 
 
 
 
 
 
 
e80aab9
 
0e8fcd1
a5b7567
14b9969
3de0945
f6f7a30
0e8fcd1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260

import os
import gradio as gr
import requests
import openai
from smolagents import OpenAIServerModel, DuckDuckGoSearchTool, CodeAgent, WikipediaSearchTool
from pathlib import Path
import tempfile
from smolagents.tools import Tool
import pathlib
from typing import Union, Optional
import pandas as pd
from tabulate import tabulate  # pragma: no cover – fallback path
import re

# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"


class SpeechToTextTool(Tool):
    """
    Transcribes an audio file to text using the OpenAI Whisper API.
    Only local file paths are supported.
    """
    name = "transcriber"
    description = (
        "This tool sends an audio file to OpenAI Whisper and returns the "
        "transcribed text."
    )
    inputs = {
        "audio": {
            "type": "string",
            "description": "Absolute or relative path to a local audio file.",
        }
    }
    output_type = "string"

    def forward(self, audio: str) -> str:
        path = Path(audio).expanduser().resolve()
        if not path.is_file():
            raise FileNotFoundError(f"No such audio file: {path}")
        with path.open("rb") as fp:
            response = openai.audio.transcriptions.create(
                file=fp,
                model="whisper-1",
                response_format="text"
            )
        return response


class ExcelToTextTool(Tool):
    """Render an Excel worksheet as Markdown text."""
    name = "excel_to_text"
    description = (
        "Read an Excel file and return a Markdown table of the requested sheet. "
        "Accepts either the sheet name or the zero-based index."
    )
    inputs = {
        "excel_path": {
            "type": "string",
            "description": "Path to the Excel file (.xlsx / .xls).",
        },
        "sheet_name": {
            "type": "string",
            "description": "Worksheet name or zero‑based index as a string (optional).",
            "nullable": True,
        },
    }
    output_type = "string"

    def forward(
        self,
        excel_path: str,
        sheet_name: Optional[str] = None,
    ) -> str:
        path = pathlib.Path(excel_path).expanduser().resolve()
        if not path.exists():
            return f"Error: Excel file not found at {path}"
        try:
            sheet = 0 if not sheet_name or sheet_name == "" else (
                int(sheet_name) if sheet_name.isdigit() else sheet_name
            )
            df = pd.read_excel(path, sheet_name=sheet)
            if hasattr(df, "to_markdown"):
                return df.to_markdown(index=False)
            return tabulate(df, headers="keys", tablefmt="github", showindex=False)
        except Exception as exc:
            return f"Error reading Excel file: {exc}"


def download_file_if_any(base_api_url: str, task_id: str) -> Optional[str]:
    """
    Try GET /files/{task_id}.
    • On HTTP 200 → save to temp dir and return local path.
    • On 404 → return None.
    """
    url = f"{base_api_url}/files/{task_id}"
    resp = requests.get(url, timeout=30)
    if resp.status_code == 404:
        return None
    resp.raise_for_status()
    # save to temp
    cdisp = resp.headers.get("content-disposition", "")
    filename = task_id
    if "filename=" in cdisp:
        m = re.search(r'filename="([^"]+)"', cdisp)
        if m:
            filename = m.group(1)
    tmp_dir = Path(tempfile.gettempdir()) / "gaia_files"
    tmp_dir.mkdir(exist_ok=True)
    file_path = tmp_dir / filename
    with open(file_path, "wb") as f:
        f.write(resp.content)
    return str(file_path)


# --- Basic Agent Definition ---
class BasicAgent:
    def __init__(self):
        self.agent = CodeAgent(
            model=OpenAIServerModel(model_id="gpt-4o"),
            tools=[
                DuckDuckGoSearchTool(),
                WikipediaSearchTool(),
                SpeechToTextTool(),
                ExcelToTextTool()
            ],
            add_base_tools=True,
            additional_authorized_imports=['pandas', 'numpy', 'csv', 'subprocess']
        )
        print("BasicAgent initialized.")

    def __call__(self, question: str) -> str:
        print(f"Agent received question (first 50 chars): {question[:50]}...")
        raw = self.agent.run(question)
        clean = self._clean(raw)
        print(f"Agent returning cleaned answer: {clean}")
        return clean

    def _clean(self, text: str) -> str:
        """
        Post-process the raw agent output to extract just the final answer.
        1) If 'FINAL ANSWER:' appears, take what follows.
        2) Otherwise, take the last non-blank line.
        3) Strip trailing periods.
        """
        # 1) look for "FINAL ANSWER:" marker
        m = re.search(r'FINAL ANSWER:\s*(.+)', text, re.IGNORECASE)
        if m:
            ans = m.group(1).strip()
        else:
            lines = [l.strip() for l in text.strip().splitlines() if l.strip()]
            ans = lines[-1] if lines else text.strip()
        # 2) remove any trailing period
        return ans.rstrip('.')


def run_and_submit_all(username: str):
    """
    Fetches questions, runs BasicAgent on each, submits answers, and returns
    (status_message, results_df).
    """
    if not username or not username.strip():
        return "Error: Please enter your username.", None

    api_url = DEFAULT_API_URL
    questions_url = f"{api_url}/questions"
    submit_url = f"{api_url}/submit"

    try:
        agent = BasicAgent()
    except Exception as e:
        return f"Error initializing agent: {e}", None

    # fetch questions
    try:
        resp = requests.get(questions_url, timeout=15)
        resp.raise_for_status()
        questions = resp.json()
    except Exception as e:
        return f"Error fetching questions: {e}", None

    results = []
    answers_payload = []

    for item in questions:
        tid = item.get("task_id")
        qtext = item.get("question")
        if not tid or not qtext:
            continue

        # download file if any
        try:
            fp = download_file_if_any(api_url, tid)
        except Exception as e:
            fp = None

        prompt = qtext
        if fp:
            prompt += f"\n\n---\nA file was downloaded and saved at:\n{fp}\n---\n"

        try:
            ans = agent(prompt)
        except Exception as e:
            ans = f"AGENT ERROR: {e}"
        results.append({
            "Task ID": tid,
            "Question": qtext,
            "Submitted Answer": ans
        })
        answers_payload.append({"task_id": tid, "submitted_answer": ans})

    if not answers_payload:
        return "Agent produced no answers.", pd.DataFrame(results)

    # submit
    submission = {
        "username": username.strip(),
        "agent_code": f"https://huggingface.co/spaces/l3xv/Final_Assignment_Template/tree/main",
        "answers": answers_payload
    }
    try:
        r2 = requests.post(submit_url, json=submission, timeout=60)
        r2.raise_for_status()
        res = r2.json()
        status = (
            f"Submission Successful!\n"
            f"User: {res.get('username')}\n"
            f"Score: {res.get('score', 'N/A')}% "
            f"({res.get('correct_count', '?')}/{res.get('total_attempted', '?')})"
        )
    except Exception as e:
        status = f"Submission Failed: {e}"

    return status, pd.DataFrame(results)


# --- Gradio UI ---
with gr.Blocks() as demo:
    gr.Markdown("# GAIA Agent Evaluation Runner")
    gr.Markdown("Enter your Hugging Face username below, then click **Run Evaluation & Submit All Answers**")
    username_input = gr.Textbox(label="Username", placeholder="hf_username")
    run_btn = gr.Button("Run Evaluation & Submit All Answers")
    status_box = gr.Textbox(label="Status / Submission Result", interactive=False, lines=3)
    results_tbl = gr.DataFrame(label="Questions & Answers", wrap=True)

    run_btn.click(
        fn=run_and_submit_all,
        inputs=[username_input],
        outputs=[status_box, results_tbl],
    )

if __name__ == "__main__":
    print("Launching GAIA Agent App…")
    demo.launch(debug=True, share=False)