File size: 13,361 Bytes
10e9b7d
d87cd81
 
 
 
 
 
 
 
10e9b7d
eccf8e4
3c4371f
d87cd81
 
10e9b7d
e80aab9
3db6293
d87cd81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e80aab9
d87cd81
31243f4
d87cd81
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3c4371f
7e4a06b
d87cd81
3c4371f
7e4a06b
7d65c66
3c4371f
d87cd81
 
 
 
 
e80aab9
31243f4
d87cd81
31243f4
 
36ed51a
3c4371f
eccf8e4
31243f4
7d65c66
31243f4
 
d87cd81
7d65c66
d87cd81
 
 
31243f4
 
 
d87cd81
31243f4
d87cd81
 
31243f4
d87cd81
 
 
 
 
 
 
 
 
 
31243f4
 
 
 
7d65c66
e80aab9
7d65c66
e80aab9
 
31243f4
e80aab9
 
3c4371f
 
 
e80aab9
d87cd81
e80aab9
d87cd81
 
7d65c66
d87cd81
e80aab9
 
d87cd81
e80aab9
31243f4
0ee0419
e514fd7
 
 
d87cd81
 
 
e514fd7
e80aab9
7e4a06b
31243f4
9088b99
7d65c66
d87cd81
e80aab9
 
d87cd81
3c4371f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
import os
import re
import sys
import base64
import tempfile
import subprocess
import threading
import concurrent.futures as cf

import gradio as gr
import requests
import pandas as pd
import anthropic
from bs4 import BeautifulSoup

# --- Constants ---
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
MODEL = os.getenv("CLAUDE_MODEL", "claude-sonnet-5-5")
MAX_STEPS = 14          # tool-use rounds per question
MAX_WORKERS = 3         # questions answered in parallel (lower it if you hit rate limits)
CHUNK = 12000           # characters returned per page fetch

IMAGE_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
               ".gif": "image/gif", ".webp": "image/webp"}
AUDIO_EXT = (".mp3", ".wav", ".m4a", ".flac", ".ogg")

SYSTEM = """You are a careful research agent solving questions from the GAIA benchmark.
Answers are graded by exact match, so the format matters as much as the content.

Method:
- Think step by step and use the tools whenever a fact is not certain. Do not guess from memory
  when a lookup is possible. For counting, sorting and arithmetic, use run_python.
- Some questions come with an attached file; its local path is given in the message.
  Use read_file (text, Excel, audio transcription) or run_python on it. Images are attached directly.
- Wikipedia pages: fetch_url works well. For tables, run_python with pandas.read_html is more reliable.
- Respect every detail of the question (date ranges, "inclusive", "only first name", alphabetical order, etc.).
- If a question is a puzzle (reversed text, logic table), solve it directly and then apply the instruction it contains.

Output rules:
- End your reply with one line: FINAL ANSWER: <answer>
- The answer must be only the requested value: a number, a short phrase, or a comma-separated list.
- No units, currency symbols or thousands separators unless the question asks for them.
- No articles or abbreviations in strings unless asked. Write numbers as digits.
- If you truly cannot find the answer after trying, still give your best single guess."""

# --- Tools ---------------------------------------------------------------

TOOLS = [
    {"type": "web_search_20250305", "name": "web_search", "max_uses": 6},
    {
        "name": "fetch_url",
        "description": "Download a web page and return its text content (up to ~12000 characters per call). "
                       "Use `start` to read further into long pages.",
        "input_schema": {
            "type": "object",
            "properties": {"url": {"type": "string"}, "start": {"type": "integer", "description": "Character offset, default 0"}},
            "required": ["url"],
        },
    },
    {
        "name": "run_python",
        "description": "Run Python 3 code (pandas, requests, bs4 available) and return stdout/stderr. "
                       "Always print() what you need. 60 second limit.",
        "input_schema": {"type": "object", "properties": {"code": {"type": "string"}}, "required": ["code"]},
    },
    {
        "name": "read_file",
        "description": "Read a local file: text/code/csv/json, Excel (all sheets as CSV), or audio (transcribed to text).",
        "input_schema": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
    },
    {
        "name": "youtube_transcript",
        "description": "Get the transcript (subtitles) of a YouTube video. Cannot see the images of the video.",
        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
    },
]

_whisper = None
_whisper_lock = threading.Lock()


def transcribe(path: str) -> str:
    global _whisper
    with _whisper_lock:
        if _whisper is None:
            from faster_whisper import WhisperModel
            _whisper = WhisperModel("base", device="cpu", compute_type="int8")
        segments, _ = _whisper.transcribe(path)
        return " ".join(s.text.strip() for s in segments)


def tool_fetch_url(url: str, start: int = 0) -> str:
    r = requests.get(url, headers={"User-Agent": "Mozilla/5.0 (compatible; AgentCourseBot/1.0)"}, timeout=25)
    r.raise_for_status()
    if "html" in r.headers.get("content-type", ""):
        soup = BeautifulSoup(r.text, "html.parser")
        for t in soup(["script", "style", "nav", "footer"]):
            t.decompose()
        text = re.sub(r"\n{2,}", "\n", soup.get_text("\n"))
    else:
        text = r.text
    return text[start:start + CHUNK] or "(no more content)"


def tool_run_python(code: str) -> str:
    with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f:
        f.write(code)
    env = {k: v for k, v in os.environ.items() if k not in ("ANTHROPIC_API_KEY", "HF_TOKEN")}
    try:
        p = subprocess.run([sys.executable, f.name], capture_output=True, text=True, timeout=60, env=env)
    except subprocess.TimeoutExpired:
        return "Timed out after 60 seconds."
    out = p.stdout + (f"\nSTDERR:\n{p.stderr}" if p.stderr else "")
    return out[:8000] or "(no output)"


def tool_read_file(path: str) -> str:
    ext = os.path.splitext(path)[1].lower()
    if ext in AUDIO_EXT:
        return transcribe(path)
    if ext in (".xlsx", ".xls"):
        sheets = pd.read_excel(path, sheet_name=None)
        return "\n\n".join(f"## Sheet: {n}\n{df.to_csv(index=False)}" for n, df in sheets.items())[:15000]
    with open(path, "r", errors="replace") as f:
        return f.read()[:15000]


def tool_youtube_transcript(url: str) -> str:
    from youtube_transcript_api import YouTubeTranscriptApi
    m = re.search(r"(?:v=|youtu\.be/)([\w-]{11})", url)
    if not m:
        return "Could not find a video id in that URL."
    data = YouTubeTranscriptApi().fetch(m.group(1))
    return "\n".join(f"[{int(s.start)}s] {s.text}" for s in data)[:15000]


def run_tool(name: str, args: dict) -> str:
    try:
        if name == "fetch_url":
            return tool_fetch_url(**args)
        if name == "run_python":
            return tool_run_python(**args)
        if name == "read_file":
            return tool_read_file(**args)
        if name == "youtube_transcript":
            return tool_youtube_transcript(**args)
        return f"Unknown tool {name}"
    except Exception as e:
        return f"Tool error: {type(e).__name__}: {e}"


# --- Agent ---------------------------------------------------------------

def extract_final(resp) -> str:
    text = "".join(b.text for b in resp.content if b.type == "text").strip()
    matches = re.findall(r"FINAL ANSWER:\s*(.+)", text)
    answer = matches[-1].strip() if matches else text.splitlines()[-1].strip() if text else ""
    return answer.strip().strip("*").rstrip(".") if answer else ""


class ClaudeAgent:
    def __init__(self):
        self.client = anthropic.Anthropic(max_retries=6)
        print(f"ClaudeAgent initialized with model {MODEL}")

    def __call__(self, question: str, file_path: str | None = None) -> str:
        content = []
        text = question
        if file_path:
            ext = os.path.splitext(file_path)[1].lower()
            if ext in IMAGE_TYPES:
                with open(file_path, "rb") as f:
                    data = base64.standard_b64encode(f.read()).decode()
                content.append({"type": "image", "source": {"type": "base64", "media_type": IMAGE_TYPES[ext], "data": data}})
                text += "\n\n(The attached image is shown above.)"
            else:
                text += f"\n\n[Attached file saved locally at: {file_path}]"
        content.append({"type": "text", "text": text})
        messages = [{"role": "user", "content": content}]

        resp = None
        for _ in range(MAX_STEPS):
            resp = self.client.messages.create(
                model=MODEL, max_tokens=3000, system=SYSTEM, tools=TOOLS, messages=messages
            )
            messages.append({"role": "assistant", "content": resp.content})
            if resp.stop_reason == "tool_use":
                results = [
                    {"type": "tool_result", "tool_use_id": b.id, "content": run_tool(b.name, b.input)}
                    for b in resp.content if b.type == "tool_use"
                ]
                messages.append({"role": "user", "content": results})
            elif resp.stop_reason == "pause_turn":
                continue
            else:
                break

        # If we ran out of steps without a final answer, force one.
        full = "".join(b.text for b in resp.content if b.type == "text") if resp else ""
        if "FINAL ANSWER:" not in full:
            messages.append({"role": "user", "content": "Stop using tools. Give your best final answer now, ending with 'FINAL ANSWER: <answer>'."})
            resp = self.client.messages.create(model=MODEL, max_tokens=500, system=SYSTEM, messages=messages)
        return extract_final(resp)


def download_task_file(task_id: str, file_name: str) -> str | None:
    if not file_name:
        return None
    try:
        r = requests.get(f"{DEFAULT_API_URL}/files/{task_id}", timeout=30)
        if r.status_code != 200 or not r.content:
            return None
        path = os.path.join(tempfile.gettempdir(), f"{task_id}_{file_name}")
        with open(path, "wb") as f:
            f.write(r.content)
        return path
    except Exception as e:
        print(f"Could not download file for {task_id}: {e}")
        return None


def run_and_submit_all(profile: gr.OAuthProfile | None):
    space_id = os.getenv("SPACE_ID")

    if profile:
        username = f"{profile.username}"
        print(f"User logged in: {username}")
    else:
        return "Please Login to Hugging Face with the button.", None

    if not os.getenv("ANTHROPIC_API_KEY"):
        return "Missing ANTHROPIC_API_KEY. Add it as a secret in the Space settings.", None

    questions_url = f"{DEFAULT_API_URL}/questions"
    submit_url = f"{DEFAULT_API_URL}/submit"

    try:
        agent = ClaudeAgent()
    except Exception as e:
        return f"Error initializing agent: {e}", None
    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"

    try:
        response = requests.get(questions_url, timeout=15)
        response.raise_for_status()
        questions_data = response.json()
        if not questions_data:
            return "Fetched questions list is empty or invalid format.", None
    except Exception as e:
        return f"Error fetching questions: {e}", None

    def solve(item):
        task_id = item.get("task_id")
        question_text = item.get("question")
        if not task_id or question_text is None:
            return None
        try:
            path = download_task_file(task_id, item.get("file_name"))
            answer = agent(question_text, path)
        except Exception as e:
            print(f"Error on task {task_id}: {e}")
            answer = f"AGENT ERROR: {e}"
        print(f"[{task_id}] -> {answer}")
        return task_id, question_text, answer

    with cf.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
        outputs = [o for o in pool.map(solve, questions_data) if o]

    results_log = [{"Task ID": t, "Question": q, "Submitted Answer": a} for t, q, a in outputs]
    answers_payload = [{"task_id": t, "submitted_answer": a} for t, q, a in outputs if not a.startswith("AGENT ERROR")]

    if not answers_payload:
        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)

    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
    try:
        response = requests.post(submit_url, json=submission_data, timeout=60)
        response.raise_for_status()
        result_data = response.json()
        final_status = (
            f"Submission Successful!\n"
            f"User: {result_data.get('username')}\n"
            f"Overall Score: {result_data.get('score', 'N/A')}% "
            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
            f"Message: {result_data.get('message', 'No message received.')}"
        )
        return final_status, pd.DataFrame(results_log)
    except requests.exceptions.HTTPError as e:
        detail = f"Server responded with status {e.response.status_code}. Response: {e.response.text[:500]}"
        return f"Submission Failed: {detail}", pd.DataFrame(results_log)
    except Exception as e:
        return f"Submission Failed: {e}", pd.DataFrame(results_log)


# --- Gradio interface ---
with gr.Blocks() as demo:
    gr.Markdown("# Basic Agent Evaluation Runner")
    gr.Markdown(
        """
        **Instructions:**

        1. Add your `ANTHROPIC_API_KEY` as a secret in the Space settings.
        2. Log in to your Hugging Face account using the button below.
        3. Click 'Run Evaluation & Submit All Answers'. It can take 10 to 20 minutes.
        """
    )
    gr.LoginButton()
    run_button = gr.Button("Run Evaluation & Submit All Answers")
    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
    run_button.click(fn=run_and_submit_all, outputs=[status_output, results_table])

if __name__ == "__main__":
    print("Launching Gradio Interface for Agent Evaluation...")
    demo.launch(debug=True, share=False)