Spaces:
Sleeping
Sleeping
| import os | |
| import gradio as gr | |
| import requests | |
| import pandas as pd | |
| import re | |
| import base64 | |
| import subprocess | |
| import tempfile | |
| import time | |
| from smolagents import ( | |
| CodeAgent, | |
| ToolCallingAgent, | |
| DuckDuckGoSearchTool, | |
| OpenAIModel, | |
| Tool, | |
| WikipediaSearchTool, | |
| PythonInterpreterTool, | |
| InferenceClientModel | |
| ) | |
| # --- Constants --- | |
| DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" | |
| MODEL_NAME = "gemini-2.5-flash" | |
| #MODEL_NAME = "gemini-3.5-flash" | |
| API_BASE = "https://generativelanguage.googleapis.com/v1beta/openai/" | |
| # --- System Prompt --- | |
| SYSTEM_PROMPT = """ | |
| You are a GAIA benchmark agent. Your sole objective is to produce the correct final answer. | |
| === OUTPUT RULES === | |
| - Return ONLY the final answer — no explanation, no preamble, no formatting, no trailing period | |
| - Numbers: digits only (e.g. 42, not "forty-two"), no commas, no units unless the question asks for them | |
| - Names: exact name only | |
| - Lists: comma-separated if required by the question | |
| - Dates: use the format explicitly requested, or ISO (YYYY-MM-DD) if unspecified | |
| === REASONING STRATEGY === | |
| 1. PLAN FIRST: Before calling any tool, reason about what information is needed and which tool is best suited. Keep reasoning BRIEF (1-2 sentences max). | |
| 2. DECOMPOSE: Break multi-step questions into sub-tasks. Solve each sub-task before combining into a final answer. | |
| 3. VERIFY: Cross-check answers using a second tool or source when feasible. Prefer primary/authoritative sources. But do NOT obsess — one confident answer is better than endless second-guessing. | |
| 4. DEDUCE: If full content is unavailable (paywalls, broken links, restricted video), use titles, descriptions, metadata, and search snippets to logically infer the answer. | |
| 5. STOP OVERTHINKING: Do not reason about your reasoning. Once you have the answer, output it immediately. | |
| === TOOL USAGE RULES === | |
| 0. WIKIPEDIA TABLES: If the answer is in a Wikipedia list or table (discography, studio albums, filmography, awards, sports stats, election results), plain-text search will NOT show it — call wikipedia_tables(page) instead. If a section header appears but its content is empty, that content was a table: use wikipedia_tables. | |
| 1. NO REPEAT CALLS: Never call the same tool with the same arguments twice. If a call fails or returns empty, move on. | |
| 2. FALLBACK CHAIN: Specialized tool fails → general web search → page fetch → deduce from context. | |
| 3. ONE FALLBACK: Use each fallback strategy exactly once per sub-task. | |
| 4. STOP LOOPING: If after 3 distinct tool attempts the answer is still unclear, make your best-reasoned guess and output it immediately. | |
| === FAILURE RECOVERY === | |
| - Stuck or hitting max steps? Output your single best answer NOW and stop. | |
| - Partial information is enough — reason from what you have. | |
| - An educated, well-reasoned guess beats silence or an infinite loop. | |
| """ | |
| WEB_HEADERS = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/122 Safari/537.36", | |
| "Accept-Language": "en-US,en;q=0.9", | |
| "Referer": "https://www.google.com/", | |
| } | |
| # ───────────────────────────────────────────────────────────── | |
| # Answer cleaning — GAIA scoring is (near) exact-match | |
| # ───────────────────────────────────────────────────────────── | |
| def clean_answer(raw) -> str: | |
| if raw is None or raw == "": | |
| return "NO_ANSWER" | |
| ans = str(raw).strip() | |
| if not ans or ans.lower() in ("none", "error", "failed"): | |
| return "NO_ANSWER" | |
| # Strip common prefixes the model sometimes adds | |
| ans = re.sub(r"^(final answer|answer|the answer is)\s*[:\-]?\s*", "", ans, flags=re.I).strip() | |
| # Strip surrounding quotes / markdown / trailing period | |
| ans = ans.strip("`").strip() | |
| if len(ans) > 1: | |
| ans = ans.strip('"').strip("'") | |
| ans = ans.rstrip(".") | |
| final = ans.strip() | |
| return final if final else "NO_ANSWER" | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – YouTube transcript | |
| # ───────────────────────────────────────────────────────────── | |
| class YouTubeTranscriptTool(Tool): | |
| name = "youtube_transcript" | |
| description = ( | |
| "Fetch the transcript (captions) of a YouTube video. " | |
| "Use whenever a question references a YouTube URL and asks about " | |
| "video content, dialogue, or what is shown." | |
| ) | |
| inputs = {"url": {"type": "string", "description": "Full YouTube video URL"}} | |
| output_type = "string" | |
| def forward(self, url: str) -> str: | |
| m = re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]{11})", url) | |
| if not m: | |
| return f"Could not extract video ID from: {url}" | |
| vid = m.group(1) | |
| headers = {"User-Agent": "Mozilla/5.0"} | |
| # Primary: youtubetotranscript.com | |
| try: | |
| r = requests.get( | |
| f"https://youtubetotranscript.com/transcript?v={vid}&lang=en", | |
| headers=headers, timeout=8, | |
| ) | |
| if r.status_code == 200 and len(r.text.strip()) > 50: | |
| text = re.sub(r"<[^>]+>", " ", r.text) | |
| text = re.sub(r"\s{2,}", " ", text).strip() | |
| return text[:12000] | |
| except Exception: | |
| pass | |
| # Fallback: public proxy JSON API | |
| try: | |
| r = requests.get( | |
| f"https://api.youtubetranscript.com/?video_id={vid}", | |
| headers=headers, timeout=8, | |
| ) | |
| if r.status_code == 200: | |
| segs = r.json() | |
| text = " ".join(s.get("text", "") for s in segs if isinstance(s, dict)) | |
| if text: | |
| return text[:12000] | |
| except Exception: | |
| pass | |
| # Fallback 2: yt-dlp subtitle fetch (if installed in the Space) | |
| try: | |
| subprocess.run( | |
| ["yt-dlp", "--skip-download", "--write-auto-sub", | |
| "--sub-lang", "en", "--sub-format", "vtt", | |
| "-o", "/tmp/ytsub", url], | |
| capture_output=True, text=True, timeout=12, | |
| ) | |
| for f in ["/tmp/ytsub.en.vtt", "/tmp/ytsub.en-US.vtt"]: | |
| if os.path.exists(f): | |
| raw = open(f).read() | |
| clean = re.sub(r"<[^>]+>", "", raw) | |
| clean = re.sub(r"\d{2}:\d{2}.*\n", "", clean) | |
| clean = re.sub(r"\n{2,}", "\n", clean).strip() | |
| return clean[:12000] | |
| except Exception: | |
| pass | |
| return ( | |
| f"Could not retrieve transcript for video {vid}. " | |
| "Try searching for the video title + 'transcript' via web search." | |
| ) | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Safe webpage visitor (strips HTML tags) | |
| # ───────────────────────────────────────────────────────────── | |
| class SafeVisitWebpageTool(Tool): | |
| name = "visit_webpage" | |
| description = "Fetch a webpage and return its readable plain-text content." | |
| inputs = {"url": {"type": "string", "description": "URL to fetch"}} | |
| output_type = "string" | |
| def forward(self, url: str) -> str: | |
| try: | |
| r = requests.get(url, headers=WEB_HEADERS, timeout=15, allow_redirects=True) | |
| if r.status_code != 200: | |
| return f"HTTP {r.status_code}" | |
| # Drop script/style blocks before stripping tags | |
| text = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", r.text) | |
| text = re.sub(r"<[^>]+>", " ", text) | |
| text = re.sub(r"[ \t]{2,}", " ", text) | |
| text = re.sub(r"\n{3,}", "\n\n", text).strip() | |
| return text[:10000] | |
| except Exception as e: | |
| return f"ERROR: {e}" | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Robust web search (DDG with retries + Wikipedia fallback) | |
| # ───────────────────────────────────────────────────────────── | |
| class RobustSearchTool(Tool): | |
| name = "web_search" | |
| description = ( | |
| "Search the web and return top results (titles, snippets, URLs). " | |
| "Automatically retries on rate limits." | |
| ) | |
| inputs = {"query": {"type": "string", "description": "Search query"}} | |
| output_type = "string" | |
| def __init__(self): | |
| super().__init__() | |
| self._ddg = DuckDuckGoSearchTool() | |
| def forward(self, query: str) -> str: | |
| last_err = "" | |
| for attempt in range(3): | |
| try: | |
| res = self._ddg.forward(query) | |
| if res and len(str(res).strip()) > 20: | |
| return str(res) | |
| except Exception as e: | |
| last_err = str(e) | |
| time.sleep(4 * (attempt + 1)) # back off on DDG rate limits | |
| return f"Search failed after retries ({last_err}). Try wikipedia_search or visit_webpage instead." | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Wikipedia table extractor (discographies, filmographies, | |
| # sports stats — anything stored as an HTML table) | |
| # ───────────────────────────────────────────────────────────── | |
| class WikipediaTablesTool(Tool): | |
| name = "wikipedia_tables" | |
| description = ( | |
| "Extract ALL tables from a Wikipedia page as markdown. " | |
| "USE THIS whenever the answer lives in a table/list on Wikipedia: " | |
| "discographies (studio albums), filmographies, award lists, sports " | |
| "statistics, election results, etc. Plain-text Wikipedia search does " | |
| "NOT return tables — this tool does." | |
| ) | |
| inputs = { | |
| "page": { | |
| "type": "string", | |
| "description": "Wikipedia page title (e.g. 'Mercedes Sosa') or full URL", | |
| }, | |
| "keyword": { | |
| "type": "string", | |
| "description": "Optional keyword to filter tables (e.g. 'album', 'year'). Empty = return all tables.", | |
| "default": "", "nullable": True, | |
| }, | |
| } | |
| output_type = "string" | |
| def forward(self, page: str, keyword: str = "") -> str: | |
| if page.startswith("http"): | |
| url = page | |
| else: | |
| url = "https://en.wikipedia.org/wiki/" + page.replace(" ", "_") | |
| try: | |
| r = requests.get(url, headers=WEB_HEADERS, timeout=20) | |
| r.raise_for_status() | |
| import io | |
| tables = pd.read_html(io.StringIO(r.text)) | |
| except Exception as e: | |
| return f"Could not extract tables from {url}: {e}" | |
| if not tables: | |
| return f"No tables found on {url}" | |
| out = [] | |
| for i, df in enumerate(tables): | |
| try: | |
| md = df.to_markdown(index=False) | |
| except Exception: | |
| continue | |
| if keyword and keyword.lower() not in md.lower(): | |
| continue | |
| out.append(f"### Table {i} ({df.shape[0]} rows x {df.shape[1]} cols)\n{md}") | |
| if not out: | |
| return f"No tables matching '{keyword}' found on {url}. Retry with empty keyword." | |
| result = "\n\n".join(out) | |
| return result[:25000] + ("\n... (truncated)" if len(result) > 25000 else "") | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Read text-based files (py, txt, csv, json, md...) | |
| # ───────────────────────────────────────────────────────────── | |
| class ReadTextFileTool(Tool): | |
| name = "read_text_file" | |
| description = ( | |
| "Read a local text-based file (.py, .txt, .csv, .json, .md, .html) and return its content. " | |
| "Use this to inspect attached Python scripts or data files before reasoning about them." | |
| ) | |
| inputs = {"path": {"type": "string", "description": "Local file path"}} | |
| output_type = "string" | |
| def forward(self, path: str) -> str: | |
| try: | |
| with open(path, "r", encoding="utf-8", errors="replace") as f: | |
| content = f.read() | |
| return content[:15000] | |
| except Exception as e: | |
| return f"Could not read file: {e}" | |
| from huggingface_hub import InferenceClient | |
| class AnalyzeImageTool(Tool): | |
| name = "analyze_image" | |
| description = ( | |
| "Answer questions about images by analyzing their visual content. " | |
| "Use for chess positions, diagrams, photographs, charts, or any visual material. " | |
| "Provide the image URL or local path and your question." | |
| ) | |
| inputs = { | |
| "source": {"type": "string", "description": "Image URL or local file path (PNG/JPG/GIF/WEBP)"}, | |
| "question": {"type": "string", "description": "What do you want to know about the image?"}, | |
| } | |
| output_type = "string" | |
| def __init__(self): | |
| self.client = InferenceClient( | |
| model="Qwen/Qwen2-VL-7B-Instruct", | |
| token=os.getenv("HF_TOKEN") | |
| ) | |
| def forward(self, source: str, question: str) -> str: | |
| try: | |
| if source.startswith("http"): | |
| response = self.client.visual_question_answering( | |
| image=source, | |
| question=question | |
| ) | |
| else: | |
| with open(source, "rb") as f: | |
| response = self.client.visual_question_answering( | |
| image=f.read(), | |
| question=question | |
| ) | |
| return response | |
| except Exception as e: | |
| return f"Vision error: {e}" | |
| class TranscribeAudioTool(Tool): | |
| name = "transcribe_audio" | |
| description = ( | |
| "Convert speech from audio files into text. " | |
| "Use for podcasts, interviews, voice memos, or any recorded speech content." | |
| ) | |
| inputs = { | |
| "source": { | |
| "type": "string", | |
| "description": "Audio file URL or local path (MP3/WAV/M4A/FLAC/OGG)", | |
| }, | |
| } | |
| output_type = "string" | |
| def __init__(self): | |
| from huggingface_hub import InferenceClient | |
| self.client = InferenceClient( | |
| model="openai/whisper-small", | |
| token=os.getenv("HF_TOKEN") | |
| ) | |
| def forward(self, source: str) -> str: | |
| try: | |
| if source.startswith("http"): | |
| r = requests.get(source, timeout=30) | |
| r.raise_for_status() | |
| audio_data = r.content | |
| else: | |
| with open(source, "rb") as f: | |
| audio_data = f.read() | |
| response = self.client.automatic_speech_recognition(audio_data) | |
| return response.get("text", "No speech detected") | |
| except Exception as e: | |
| return f"Transcription error: {e}" | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Excel file reader | |
| # ───────────────────────────────────────────────────────────── | |
| class ReadExcelFileTool(Tool): | |
| name = "read_excel_file" | |
| description = ( | |
| "Download an Excel (.xlsx) file and return its contents as a markdown table. " | |
| "Use when a question involves data from an attached spreadsheet." | |
| ) | |
| inputs = { | |
| "source": { | |
| "type": "string", | |
| "description": "URL or local file path of the .xlsx file", | |
| }, | |
| "sheet": { | |
| "type": "string", | |
| "description": "Sheet name or index to read (default: first sheet)", | |
| "default": "0", "nullable": True, | |
| }, | |
| } | |
| output_type = "string" | |
| def forward(self, source: str, sheet: str = "0") -> str: | |
| try: | |
| if source.startswith("http"): | |
| r = requests.get(source, timeout=15) | |
| r.raise_for_status() | |
| with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp: | |
| tmp.write(r.content) | |
| tmp_path = tmp.name | |
| else: | |
| tmp_path = source | |
| try: | |
| sheet_arg = int(sheet) | |
| except (ValueError, TypeError): | |
| sheet_arg = sheet if sheet else 0 | |
| df = pd.read_excel(tmp_path, sheet_name=sheet_arg) | |
| if source.startswith("http"): | |
| os.unlink(tmp_path) | |
| md = df.to_markdown(index=False) | |
| if len(md) > 10_000: | |
| summary = f"Shape: {df.shape}\nColumns: {list(df.columns)}\n\n" | |
| return summary + md[:10_000] + "\n... (truncated)" | |
| return md | |
| except Exception as e: | |
| return f"Excel read error: {e}" | |
| # ───────────────────────────────────────────────────────────── | |
| # Tool – Task file downloader (fetches attached files from scoring API) | |
| # ───────────────────────────────────────────────────────────── | |
| class DownloadTaskFileTool(Tool): | |
| name = "download_task_file" | |
| description = ( | |
| "Download an attached file for a GAIA task from the scoring API and save it locally. " | |
| "Returns the local file path. Use this FIRST whenever a question has an attached file " | |
| "(image, audio, Python script, Excel spreadsheet, etc.), then pass the returned path to the " | |
| "appropriate analysis tool (analyze_image, transcribe_audio, read_text_file, read_excel_file)." | |
| ) | |
| inputs = { | |
| "task_id": { | |
| "type": "string", | |
| "description": "The task_id of the GAIA question whose file you want to download", | |
| }, | |
| "file_name": { | |
| "type": "string", | |
| "description": "The file_name field from the question metadata (e.g. 'abc123.png')", | |
| }, | |
| } | |
| output_type = "string" | |
| def forward(self, task_id: str, file_name: str) -> str: | |
| url = f"{DEFAULT_API_URL}/files/{task_id}" | |
| try: | |
| r = requests.get(url, timeout=5) | |
| r.raise_for_status() | |
| except requests.exceptions.HTTPError as e: | |
| # 404 = file not available; 5XX = server issue; don't retry | |
| return f"File unavailable (HTTP {e.response.status_code}). Use alternative methods." | |
| except Exception as e: | |
| return f"Could not download file: {type(e).__name__}. Try web search instead." | |
| ext = file_name.rsplit(".", 1)[-1] if "." in file_name else "bin" | |
| with tempfile.NamedTemporaryFile( | |
| suffix=f".{ext}", prefix=f"gaia_{task_id}_", | |
| delete=False, dir="/tmp", | |
| ) as tmp: | |
| tmp.write(r.content) | |
| return tmp.name | |
| # ───────────────────────────────────────────────────────────── | |
| # Agent | |
| # ───────────────────────────────────────────────────────────── | |
| class BasicAgent: | |
| def __init__(self): | |
| model = OpenAIModel( | |
| model_id=MODEL_NAME, | |
| api_key=os.getenv("GEMINI_API_KEY") or os.getenv("API_KEY"), | |
| api_base=API_BASE, | |
| ) | |
| # model = InferenceClientModel("Qwen/Qwen2.5-72B-Instruct") | |
| # ───────────────────────────── | |
| # SUB-AGENT (web research) | |
| # ───────────────────────────── | |
| web_agent = ToolCallingAgent( | |
| tools=[ | |
| RobustSearchTool(), | |
| WikipediaSearchTool( | |
| user_agent="MySmolAgentApp/1.0 (contact@example.com)", | |
| language="en", | |
| content_type="text", | |
| extract_format="WIKI", | |
| ), | |
| SafeVisitWebpageTool(), | |
| WikipediaTablesTool(), | |
| ], | |
| model=model, | |
| max_steps=10, | |
| name="web_agent", | |
| description=( | |
| "Use ONLY for web search, wikipedia and browsing. " | |
| "Always prefer wikipedia for factual queries with known entities." | |
| ), | |
| ) | |
| # ───────────────────────────── | |
| # MAIN AGENT | |
| # ───────────────────────────── | |
| self.agent = CodeAgent( | |
| tools=[ | |
| PythonInterpreterTool(), | |
| YouTubeTranscriptTool(), | |
| SafeVisitWebpageTool(), | |
| AnalyzeImageTool(), | |
| TranscribeAudioTool(), | |
| ReadExcelFileTool(), | |
| ReadTextFileTool(), | |
| WikipediaTablesTool(), | |
| DownloadTaskFileTool(), | |
| WikipediaSearchTool( | |
| user_agent="MySmolAgentApp/1.0 (contact@example.com)", | |
| language="en", | |
| content_type="text", | |
| extract_format="WIKI", | |
| ), | |
| ], | |
| model=model, | |
| managed_agents=[web_agent], | |
| additional_authorized_imports=[ | |
| "re", "json", "math", "collections", "pandas", | |
| "datetime", "statistics", "base64", "os", "itertools", | |
| ], | |
| max_steps=15, | |
| verbosity_level=1, | |
| ) | |
| def __call__(self, question: str, task_id: str = "", file_name: str = ""): | |
| prompt = SYSTEM_PROMPT + f"\n\nQuestion: {question}" | |
| if task_id and file_name: | |
| prompt += f""" | |
| ATTACHED FILE DETECTED: | |
| - task_id: {task_id} | |
| - file_name: {file_name} | |
| INSTRUCTION: | |
| 1. First call download_task_file(task_id="{task_id}", file_name="{file_name}") to get the local path. | |
| 2. Then route the file to the correct tool based on its extension: | |
| - png/jpg/jpeg/gif/webp → analyze_image(path, question) | |
| - mp3/wav/m4a/ogg/flac → transcribe_audio(path) | |
| - xlsx/xls → read_excel_file(path) | |
| - py/txt/csv/json/md → read_text_file(path), then REASON about the content | |
| (for .py files: read the code, trace its logic mentally or re-implement it | |
| with python_interpreter — do NOT try to execute the downloaded file directly) | |
| 3. Use the extracted information to answer the question. | |
| """ | |
| return self.agent.run(prompt) | |
| def run_and_submit_all(profile: gr.OAuthProfile | None): | |
| """ | |
| Fetches all questions, runs the BasicAgent on them, submits all answers, | |
| and displays the results. | |
| """ | |
| space_id = os.getenv("SPACE_ID") | |
| if profile: | |
| username = f"{profile.username}" | |
| print(f"User logged in: {username}") | |
| else: | |
| print("User not logged in.") | |
| return "Please Login to Hugging Face with the button.", None | |
| api_url = DEFAULT_API_URL | |
| questions_url = f"{api_url}/questions" | |
| submit_url = f"{api_url}/submit" | |
| # 1. Instantiate Agent | |
| try: | |
| agent = BasicAgent() | |
| except Exception as e: | |
| print(f"Error instantiating agent: {e}") | |
| return f"Error initializing agent: {e}", None | |
| agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" | |
| print(agent_code) | |
| # 2. Fetch Questions | |
| print(f"Fetching questions from: {questions_url}") | |
| try: | |
| response = requests.get(questions_url, timeout=15) | |
| response.raise_for_status() | |
| questions_data = response.json() | |
| if not questions_data: | |
| print("Fetched questions list is empty.") | |
| return "Fetched questions list is empty or invalid format.", None | |
| print(f"Fetched {len(questions_data)} questions.") | |
| except requests.exceptions.RequestException as e: | |
| print(f"Error fetching questions: {e}") | |
| return f"Error fetching questions: {e}", None | |
| except requests.exceptions.JSONDecodeError as e: | |
| print(f"Error decoding JSON response from questions endpoint: {e}") | |
| print(f"Response text: {response.text[:500]}") | |
| return f"Error decoding server response for questions: {e}", None | |
| except Exception as e: | |
| print(f"An unexpected error occurred fetching questions: {e}") | |
| return f"An unexpected error occurred fetching questions: {e}", None | |
| # 3. Run the Agent | |
| results_log = [] | |
| answers_payload = [] | |
| print(f"Running agent on {len(questions_data)} questions...") | |
| for item in questions_data: | |
| task_id = item.get("task_id") | |
| question_text = item.get("question") | |
| file_name = item.get("file_name", "") or "" | |
| if not task_id or question_text is None: | |
| print(f"Skipping item with missing task_id or question: {item}") | |
| continue | |
| try: | |
| # *** FIX: pass task_id and file_name so file-based questions work *** | |
| raw_answer = agent(question_text, task_id=task_id, file_name=file_name) | |
| submitted_answer = clean_answer(raw_answer) | |
| answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) | |
| results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer}) | |
| except Exception as e: | |
| print(f"Error running agent on task {task_id}: {e}") | |
| results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"}) | |
| finally: | |
| # rate-limit cushion — runs after both success AND failure | |
| time.sleep(15) | |
| if not answers_payload: | |
| print("Agent did not produce any answers to submit.") | |
| return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) | |
| # 4. Prepare Submission | |
| submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload} | |
| status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..." | |
| print(status_update) | |
| # 5. Submit | |
| print(f"Submitting {len(answers_payload)} answers to: {submit_url}") | |
| try: | |
| response = requests.post(submit_url, json=submission_data, timeout=60) | |
| response.raise_for_status() | |
| result_data = response.json() | |
| final_status = ( | |
| f"Submission Successful!\n" | |
| f"User: {result_data.get('username')}\n" | |
| f"Overall Score: {result_data.get('score', 'N/A')}% " | |
| f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" | |
| f"Message: {result_data.get('message', 'No message received.')}" | |
| ) | |
| print("Submission successful.") | |
| results_df = pd.DataFrame(results_log) | |
| return final_status, results_df | |
| except requests.exceptions.HTTPError as e: | |
| error_detail = f"Server responded with status {e.response.status_code}." | |
| try: | |
| error_json = e.response.json() | |
| error_detail += f" Detail: {error_json.get('detail', e.response.text)}" | |
| except requests.exceptions.JSONDecodeError: | |
| error_detail += f" Response: {e.response.text[:500]}" | |
| status_message = f"Submission Failed: {error_detail}" | |
| print(status_message) | |
| results_df = pd.DataFrame(results_log) | |
| return status_message, results_df | |
| except requests.exceptions.Timeout: | |
| status_message = "Submission Failed: The request timed out." | |
| print(status_message) | |
| results_df = pd.DataFrame(results_log) | |
| return status_message, results_df | |
| except requests.exceptions.RequestException as e: | |
| status_message = f"Submission Failed: Network error - {e}" | |
| print(status_message) | |
| results_df = pd.DataFrame(results_log) | |
| return status_message, results_df | |
| except Exception as e: | |
| status_message = f"An unexpected error occurred during submission: {e}" | |
| print(status_message) | |
| results_df = pd.DataFrame(results_log) | |
| return status_message, results_df | |
| # --- Build Gradio Interface using Blocks --- | |
| with gr.Blocks() as demo: | |
| gr.Markdown("# Basic Agent Evaluation Runner") | |
| gr.Markdown( | |
| """ | |
| **Instructions:** | |
| 1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ... | |
| 2. Log in to your Hugging Face account using the button below. This uses your HF username for submission. | |
| 3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score. | |
| --- | |
| **Disclaimers:** | |
| Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions). | |
| This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async. | |
| """ | |
| ) | |
| gr.LoginButton() | |
| run_button = gr.Button("Run Evaluation & Submit All Answers") | |
| status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) | |
| results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) | |
| run_button.click( | |
| fn=run_and_submit_all, | |
| outputs=[status_output, results_table] | |
| ) | |
| if __name__ == "__main__": | |
| print("\n" + "-" * 30 + " App Starting " + "-" * 30) | |
| space_host_startup = os.getenv("SPACE_HOST") | |
| space_id_startup = os.getenv("SPACE_ID") | |
| if space_host_startup: | |
| print(f"✅ SPACE_HOST found: {space_host_startup}") | |
| print(f" Runtime URL should be: https://{space_host_startup}.hf.space") | |
| else: | |
| print("ℹ️ SPACE_HOST environment variable not found (running locally?).") | |
| if space_id_startup: | |
| print(f"✅ SPACE_ID found: {space_id_startup}") | |
| print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}") | |
| print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main") | |
| else: | |
| print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") | |
| print("-" * (60 + len(" App Starting ")) + "\n") | |
| print("Launching Gradio Interface for Basic Agent Evaluation...") | |
| demo.launch(debug=True, share=False) |