first commit

agent.py

@@ -0,0 +1,344 @@
+import os
+import re
+import base64
+import tempfile
+import requests
+
+from smolagents import CodeAgent, LiteLLMModel, tool, DuckDuckGoSearchTool
+
+API_URL = "https://agents-course-unit4-scoring.hf.space"
+
+ANSWER_FORMATTING_RULES = """
+
+## CRITICAL ANSWER FORMATTING RULES (GAIA exact-match grading)
+
+You MUST follow these rules when producing your final answer:
+
+1. Return ONLY the final answer value. No preamble, no explanation, no restated question.
+   Never write "The answer is..." or "I think..." — just the raw answer.
+2. Never include the literal string "FINAL ANSWER" anywhere in your output.
+3. Numbers: Use digits only. No commas, no currency symbols ($, €, £), no percent signs (%),
+   no unit abbreviations unless the question explicitly asks for them.
+4. Strings: Do NOT include articles (a, an, the) unless they are part of a proper noun.
+   Do not abbreviate unless the question explicitly asks for an abbreviation.
+5. Comma-separated lists: Apply the above rules to each element, join with ", ".
+6. Dates: Use the format explicitly requested. If none specified, use YYYY-MM-DD.
+7. For true/false questions, answer only "True" or "False" (no period).
+"""
+
+
+def _clean_answer(raw: str) -> str:
+    """Post-process agent output to ensure a clean, gradable answer string."""
+    if raw is None:
+        return ""
+    if not isinstance(raw, str):
+        return str(raw)
+
+    answer = raw.strip()
+
+    for prefix in [
+        "FINAL ANSWER:", "FINAL ANSWER :", "Final Answer:", "Final Answer :",
+        "final answer:", "final answer :",
+    ]:
+        if answer.startswith(prefix):
+            answer = answer[len(prefix):].strip()
+            break
+
+    if len(answer) >= 2 and answer[0] == '"' and answer[-1] == '"':
+        answer = answer[1:-1].strip()
+    if len(answer) >= 2 and answer[0] == "'" and answer[-1] == "'":
+        answer = answer[1:-1].strip()
+
+    answer = re.sub(r"\s+", " ", answer).strip()
+
+    return answer
+
+
+@tool
+def download_task_file(task_id: str) -> str:
+    """Download the file attached to a GAIA task from the scoring API.
+
+    Args:
+        task_id: The task_id string from the questions endpoint.
+    """
+    url = f"{API_URL}/files/{task_id}"
+    try:
+        resp = requests.get(url, timeout=60)
+        resp.raise_for_status()
+    except Exception as e:
+        return f"Error downloading file for task {task_id}: {e}"
+
+    content_type = resp.headers.get("content-type", "")
+    cd = resp.headers.get("content-disposition", "")
+    if "filename=" in cd:
+        filename = cd.split("filename=")[-1].strip('" ')
+    else:
+        ext_map = {
+            "pdf": ".pdf", "csv": ".csv", "json": ".json",
+            "text": ".txt", "html": ".html", "xml": ".xml",
+            "xlsx": ".xlsx", "excel": ".xlsx",
+            "png": ".png", "jpeg": ".jpg", "jpg": ".jpg",
+            "gif": ".gif", "webp": ".webp",
+            "audio": ".mp3", "mpeg": ".mp3", "wav": ".wav",
+            "mp4": ".mp4", "ogg": ".ogg", "flac": ".flac",
+        }
+        ext = ".bin"
+        for key, val in ext_map.items():
+            if key in content_type:
+                ext = val
+                break
+        filename = f"{task_id}{ext}"
+
+    tmp_dir = tempfile.mkdtemp()
+    filepath = os.path.join(tmp_dir, filename)
+    with open(filepath, "wb") as f:
+        f.write(resp.content)
+
+    return (
+        f"File downloaded successfully.\n"
+        f"Path: {filepath}\n"
+        f"Filename: {filename}\n"
+        f"Size: {len(resp.content)} bytes\n"
+        f"Content-Type: {content_type}\n\n"
+        f"Now use the read_file tool on this path, or analyze_image / transcribe_audio if it is an image or audio file."
+    )
+
+
+@tool
+def visit_webpage(url: str) -> str:
+    """Visit a webpage and return its text content. Handles common sites like Wikipedia.
+
+    Args:
+        url: The full URL of the webpage to visit.
+    """
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    try:
+        resp = requests.get(url, headers=headers, timeout=20)
+        resp.raise_for_status()
+        content_type = resp.headers.get("content-type", "")
+        if "text" not in content_type and "html" not in content_type:
+            return f"Response is not text (content-type: {content_type}). Cannot read as webpage."
+        text = resp.text
+        # Simple HTML tag stripping for readability
+        import re as _re
+        text = _re.sub(r"<script[^>]*>.*?</script>", "", text, flags=_re.DOTALL)
+        text = _re.sub(r"<style[^>]*>.*?</style>", "", text, flags=_re.DOTALL)
+        text = _re.sub(r"<[^>]+>", " ", text)
+        text = _re.sub(r"\s+", " ", text).strip()
+        if len(text) > 15000:
+            text = text[:15000] + "\n\n[Content truncated at 15000 characters]"
+        return text
+    except Exception as e:
+        return f"Error fetching the webpage: {e}"
+
+
+@tool
+def read_file(file_path: str) -> str:
+    """Read and parse a file from disk. Supports PDF, CSV, XLSX, JSON, plain text, and other text formats.
+
+    Args:
+        file_path: Absolute path to the file on disk.
+    """
+    if not os.path.exists(file_path):
+        return f"File not found: {file_path}"
+
+    ext = os.path.splitext(file_path)[1].lower()
+
+    try:
+        if ext == ".pdf":
+            from PyPDF2 import PdfReader
+            reader = PdfReader(file_path)
+            pages = []
+            for i, page in enumerate(reader.pages):
+                text = page.extract_text()
+                if text:
+                    pages.append(f"--- Page {i + 1} ---\n{text}")
+            if pages:
+                return "\n\n".join(pages)
+            return (
+                "PDF text extraction returned empty. "
+                "This PDF may be image-based. Try converting it to images and using analyze_image."
+            )
+
+        if ext == ".csv":
+            import pandas as pd
+            df = pd.read_csv(file_path)
+            return df.to_string(index=False)
+
+        if ext in (".xlsx", ".xls"):
+            import pandas as pd
+            df = pd.read_excel(file_path)
+            return df.to_string(index=False)
+
+        if ext in (".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".tiff"):
+            return (
+                f"This is an image file at: {file_path}\n"
+                "Use the analyze_image tool to examine its contents."
+            )
+
+        if ext in (".mp3", ".wav", ".ogg", ".m4a", ".flac", ".mp4", ".aac"):
+            return (
+                f"This is an audio file at: {file_path}\n"
+                "Use the transcribe_audio tool to get its text content."
+            )
+
+        if ext in (".json",):
+            with open(file_path, "r", encoding="utf-8") as f:
+                return f.read()
+
+        if ext in (".zip",):
+            import zipfile
+            with zipfile.ZipFile(file_path, "r") as z:
+                names = z.namelist()
+                result = f"ZIP archive contains {len(names)} files:\n"
+                for name in names[:50]:
+                    result += f"  - {name}\n"
+                if len(names) > 50:
+                    result += f"  ... and {len(names) - 50} more.\n"
+                return result
+
+        with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+            return f.read()
+
+    except Exception as e:
+        return f"Error reading file {file_path}: {e}"
+
+
+@tool
+def analyze_image(file_path: str, question: str = "Describe this image in detail. Extract all text, numbers, labels, and data you can see.") -> str:
+    """Analyze an image using a vision-capable LLM. Use this for photos, charts, diagrams, screenshots, etc.
+
+    Args:
+        file_path: Absolute path to the image file.
+        question: What you want to know about the image.
+    """
+    if not os.path.exists(file_path):
+        return f"File not found: {file_path}"
+
+    try:
+        import litellm
+
+        with open(file_path, "rb") as f:
+            img_bytes = f.read()
+        img_b64 = base64.b64encode(img_bytes).decode()
+
+        ext = os.path.splitext(file_path)[1].lower()
+        mime_map = {
+            ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
+            ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
+            ".tiff": "image/tiff",
+        }
+        mime = mime_map.get(ext, "image/png")
+
+        model_id = os.getenv("MODEL_ID", "gpt-4o")
+        api_key = (
+            os.getenv("OPENAI_API_KEY")
+            or os.getenv("ANTHROPIC_API_KEY")
+            or os.getenv("HF_TOKEN")
+        )
+
+        resp = litellm.completion(
+            model=model_id,
+            api_key=api_key,
+            messages=[{
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": question},
+                    {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{img_b64}"}},
+                ],
+            }],
+            temperature=0,
+        )
+        return resp.choices[0].message.content
+    except Exception as e:
+        return f"Error analyzing image: {e}"
+
+
+@tool
+def transcribe_audio(file_path: str) -> str:
+    """Transcribe an audio file to text using OpenAI Whisper API.
+
+    Args:
+        file_path: Absolute path to the audio file.
+    """
+    if not os.path.exists(file_path):
+        return f"File not found: {file_path}"
+
+    try:
+        from openai import OpenAI
+
+        api_key = os.getenv("OPENAI_API_KEY") or os.getenv("HF_TOKEN")
+        client = OpenAI(api_key=api_key) if api_key else OpenAI()
+
+        with open(file_path, "rb") as f:
+            result = client.audio.transcriptions.create(
+                model="whisper-1",
+                file=f,
+            )
+        return result.text
+    except Exception as e:
+        return f"Error transcribing audio: {e}"
+
+
+def build_question_prompt(item: dict) -> str:
+    """Build the full prompt to send to the agent, including file hints."""
+    question_text = item.get("question", "")
+    task_id = item.get("task_id", "")
+    file_name = item.get("file_name", "")
+
+    parts = [question_text]
+    parts.append(f"\n[TASK_ID: {task_id}]")
+
+    if file_name:
+        parts.append(
+            f"This task has an attached file: '{file_name}'. "
+            f"You MUST first call the download_task_file tool with task_id='{task_id}' "
+            f"to download it, then read or analyze the downloaded file before answering."
+        )
+
+    return "\n".join(parts)
+
+
+def create_agent():
+    """Create and return a configured CodeAgent for GAIA benchmark questions."""
+    model_id = os.getenv("MODEL_ID", "gpt-4o")
+    api_key = (
+        os.getenv("OPENAI_API_KEY")
+        or os.getenv("ANTHROPIC_API_KEY")
+        or os.getenv("HF_TOKEN")
+    )
+
+    model = LiteLLMModel(
+        model_id=model_id,
+        api_key=api_key,
+        api_base=os.getenv("API_BASE"),
+        temperature=0,
+    )
+
+    tools = [
+        DuckDuckGoSearchTool(),
+        visit_webpage,
+        download_task_file,
+        read_file,
+        analyze_image,
+        transcribe_audio,
+    ]
+
+    agent = CodeAgent(
+        tools=tools,
+        model=model,
+        max_steps=12,
+        additional_authorized_imports=[
+            "math", "datetime", "json", "re", "os", "collections",
+            "itertools", "functools", "string", "random",
+            "pandas", "numpy", "requests",
+        ],
+    )
+
+    agent.prompt_templates["system_prompt"] += ANSWER_FORMATTING_RULES
+
+    return agent

app.py

@@ -0,0 +1,162 @@
+import os
+import gradio as gr
+import requests
+import pandas as pd
+
+from agent import create_agent, _clean_answer, build_question_prompt
+
+DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
+
+
+def run_and_submit_all(profile: gr.OAuthProfile | None):
+    space_id = os.getenv("SPACE_ID")
+
+    if profile:
+        username = f"{profile.username}"
+        print(f"User logged in: {username}")
+    else:
+        print("User not logged in.")
+        return "Please Login to Hugging Face with the button.", None
+
+    api_url = DEFAULT_API_URL
+    questions_url = f"{api_url}/questions"
+    submit_url = f"{api_url}/submit"
+
+    try:
+        agent = create_agent()
+    except Exception as e:
+        print(f"Error instantiating agent: {e}")
+        return f"Error initializing agent: {e}", None
+
+    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
+    print(f"Agent code URL: {agent_code}")
+
+    print(f"Fetching questions from: {questions_url}")
+    try:
+        response = requests.get(questions_url, timeout=15)
+        response.raise_for_status()
+        questions_data = response.json()
+        if not questions_data:
+            print("Fetched questions list is empty.")
+            return "Fetched questions list is empty or invalid format.", None
+        print(f"Fetched {len(questions_data)} questions.")
+    except requests.exceptions.RequestException as e:
+        return f"Error fetching questions: {e}", None
+    except requests.exceptions.JSONDecodeError as e:
+        return f"Error decoding server response for questions: {e}", None
+    except Exception as e:
+        return f"An unexpected error occurred fetching questions: {e}", None
+
+    results_log = []
+    answers_payload = []
+    print(f"Running agent on {len(questions_data)} questions...")
+
+    for item in questions_data:
+        task_id = item.get("task_id")
+        question_text = item.get("question")
+        if not task_id or question_text is None:
+            print(f"Skipping item with missing task_id or question: {item}")
+            continue
+
+        prompt = build_question_prompt(item)
+
+        try:
+            raw_output = agent.run(prompt)
+            answer = _clean_answer(raw_output)
+            print(f"Task {task_id}: raw={raw_output!r}  cleaned={answer!r}")
+        except Exception as e:
+            print(f"Error running agent on task {task_id}: {e}")
+            answer = f"AGENT ERROR: {e}"
+
+        answers_payload.append({"task_id": task_id, "submitted_answer": answer})
+        results_log.append({
+            "Task ID": task_id,
+            "Question": question_text,
+            "Submitted Answer": answer,
+        })
+
+    if not answers_payload:
+        return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
+
+    submission_data = {
+        "username": username.strip(),
+        "agent_code": agent_code,
+        "answers": answers_payload,
+    }
+    status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
+    print(status_update)
+
+    print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
+    try:
+        response = requests.post(submit_url, json=submission_data, timeout=60)
+        response.raise_for_status()
+        result_data = response.json()
+        final_status = (
+            f"Submission Successful!\n"
+            f"User: {result_data.get('username')}\n"
+            f"Overall Score: {result_data.get('score', 'N/A')}% "
+            f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
+            f"Message: {result_data.get('message', 'No message received.')}"
+        )
+        print("Submission successful.")
+        results_df = pd.DataFrame(results_log)
+        return final_status, results_df
+    except requests.exceptions.HTTPError as e:
+        error_detail = f"Server responded with status {e.response.status_code}."
+        try:
+            error_json = e.response.json()
+            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
+        except requests.exceptions.JSONDecodeError:
+            error_detail += f" Response: {e.response.text[:500]}"
+        return f"Submission Failed: {error_detail}", pd.DataFrame(results_log)
+    except requests.exceptions.Timeout:
+        return "Submission Failed: The request timed out.", pd.DataFrame(results_log)
+    except requests.exceptions.RequestException as e:
+        return f"Submission Failed: Network error - {e}", pd.DataFrame(results_log)
+    except Exception as e:
+        return f"An unexpected error occurred during submission: {e}", pd.DataFrame(results_log)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# GAIA Benchmark Agent Runner")
+    gr.Markdown(
+        """
+        **Instructions:**
+        1. Log in to your Hugging Face account using the button below.
+        2. Click **Run Evaluation & Submit All Answers** to run the agent on all questions and submit.
+        3. Results and a per-question breakdown will appear below.
+
+        ---
+        **Note:** Running all questions takes several minutes. Each question triggers multi-step reasoning.
+        """
+    )
+
+    gr.LoginButton()
+    run_button = gr.Button("Run Evaluation & Submit All Answers")
+    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
+    results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
+
+    run_button.click(
+        fn=run_and_submit_all,
+        outputs=[status_output, results_table],
+    )
+
+if __name__ == "__main__":
+    print("\n" + "-" * 30 + " App Starting " + "-" * 30)
+    space_host = os.getenv("SPACE_HOST")
+    space_id = os.getenv("SPACE_ID")
+
+    if space_host:
+        print(f"SPACE_HOST: {space_host}")
+        print(f"  Runtime URL: https://{space_host}.hf.space")
+    else:
+        print("SPACE_HOST not found (running locally?).")
+
+    if space_id:
+        print(f"SPACE_ID: {space_id}")
+        print(f"  Repo URL: https://huggingface.co/spaces/{space_id}/tree/main")
+    else:
+        print("SPACE_ID not found (running locally?).")
+
+    print("-" * 60 + "\n")
+    demo.launch(debug=True, share=False)

requirements.txt

@@ -0,0 +1,10 @@
+gradio
+requests
+smolagents[litellm]
+duckduckgo-search
+ddgs
+PyPDF2
+pandas
+openpyxl
+openai
+Pillow

test_local.py

@@ -0,0 +1,91 @@
+"""
+Local test script for the GAIA benchmark agent.
+
+Fetches questions from the scoring API and runs the agent on 2-3 questions
+WITHOUT submitting answers. This lets you validate reasoning and answer
+formatting before burning submission attempts.
+
+Usage:
+    set MODEL_ID=gpt-4o
+    set OPENAI_API_KEY=sk-...
+    python test_local.py
+"""
+
+import os
+import sys
+import requests
+
+from agent import create_agent, _clean_answer, build_question_prompt
+
+
+API_URL = "https://agents-course-unit4-scoring.hf.space"
+NUM_QUESTIONS = 3
+
+
+def main():
+    print("=" * 60)
+    print("GAIA Agent — Local Test")
+    print("=" * 60)
+
+    # Check that the agent can be created
+    print("\n[1] Creating agent...")
+    try:
+        agent = create_agent()
+        print("    Agent created successfully.")
+    except Exception as e:
+        print(f"    Failed to create agent: {e}")
+        sys.exit(1)
+
+    # Fetch questions
+    print(f"\n[2] Fetching questions from {API_URL}/questions ...")
+    try:
+        resp = requests.get(f"{API_URL}/questions", timeout=15)
+        resp.raise_for_status()
+        questions = resp.json()
+    except Exception as e:
+        print(f"    Failed to fetch questions: {e}")
+        sys.exit(1)
+
+    print(f"    Got {len(questions)} total questions.")
+
+    # Prefer text-only questions first (no file attachment)
+    no_file = [q for q in questions if not q.get("file_name")]
+    with_file = [q for q in questions if q.get("file_name")]
+    selected = no_file[:NUM_QUESTIONS]
+    if len(selected) < NUM_QUESTIONS:
+        selected += with_file[: NUM_QUESTIONS - len(selected)]
+
+    print(f"    Selected {len(selected)} questions to test.\n")
+
+    # Run agent on each
+    for i, item in enumerate(selected):
+        task_id = item.get("task_id", "???")
+        question = item.get("question", "")
+        file_name = item.get("file_name", "")
+        prompt = build_question_prompt(item)
+
+        print("=" * 60)
+        print(f"Question {i + 1}/{len(selected)}")
+        print(f"  Task ID:     {task_id}")
+        print(f"  File:        {file_name or '(none)'}")
+        print(f"  Question:    {question[:200]}{'...' if len(question) > 200 else ''}")
+        print("-" * 60)
+
+        try:
+            raw_output = agent.run(prompt)
+        except Exception as e:
+            raw_output = f"ERROR: {e}"
+
+        cleaned = _clean_answer(raw_output)
+
+        print(f"  Raw output:   {raw_output!r}")
+        print(f"  Clean answer: {cleaned!r}")
+        print()
+
+    print("=" * 60)
+    print("Test complete.")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()