import json import os from pathlib import Path import gradio as gr import pandas as pd import requests from agent import DEFAULT_API_URL, GaiaAgent # If this Space happens to be running on ZeroGPU hardware, the runtime refuses to # start unless at least one function is decorated with @spaces.GPU. This agent never # needs a GPU (all inference is via external APIs / HF Inference), so this is a # harmless no-op that just satisfies that startup check. On CPU-only hardware the # `spaces` import/decorator are both safe no-ops too. try: import spaces @spaces.GPU def _zerogpu_startup_probe(): # pragma: no cover - never actually called return True except ImportError: pass CACHE_FILE = Path("answers_cache.json") def load_cache() -> dict: if CACHE_FILE.exists(): try: return json.loads(CACHE_FILE.read_text()) except Exception: # noqa: BLE001 pass return {} def save_cache(cache: dict) -> None: CACHE_FILE.write_text(json.dumps(cache, indent=2, ensure_ascii=False)) def fetch_questions(): r = requests.get(f"{DEFAULT_API_URL}/questions", timeout=30) r.raise_for_status() return r.json() def run_agent(use_cache: bool, progress=gr.Progress()): """Answer every question (skipping cached ones) and show the table.""" try: questions = fetch_questions() except Exception as e: # noqa: BLE001 return f"Error fetching questions: {e}", None cache = load_cache() if use_cache else {} try: agent = GaiaAgent() except Exception as e: # noqa: BLE001 return f"Error initializing agent: {e}", None rows = [] for i, item in enumerate(questions): task_id, q, fname = item["task_id"], item["question"], item.get("file_name", "") progress((i, len(questions)), desc=f"Question {i + 1}/{len(questions)}") if task_id in cache: ans = cache[task_id] else: try: ans = agent(q, task_id=task_id, file_name=fname) except Exception as e: # noqa: BLE001 ans = f"AGENT ERROR: {e}" cache[task_id] = ans save_cache(cache) rows.append({"Task ID": task_id, "Question": q, "Submitted Answer": ans}) return f"Answered {len(rows)} questions (cached in {CACHE_FILE}).", pd.DataFrame(rows) def submit_answers(profile: gr.OAuthProfile | None): """Submit cached answers for scoring.""" if not profile: return "Please log in to Hugging Face with the button.", None username = profile.username space_id = os.getenv("SPACE_ID", "") agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" cache = load_cache() if not cache: return "No answers cached yet – run the agent first.", None answers = [{"task_id": t, "submitted_answer": a} for t, a in cache.items()] payload = {"username": username.strip(), "agent_code": agent_code, "answers": answers} try: r = requests.post(f"{DEFAULT_API_URL}/submit", json=payload, timeout=120) r.raise_for_status() d = r.json() status = ( f"Submission Successful!\nUser: {d.get('username')}\n" f"Overall Score: {d.get('score', 'N/A')}% " f"({d.get('correct_count', '?')}/{d.get('total_attempted', '?')} correct)\n" f"Message: {d.get('message', '')}" ) except requests.exceptions.HTTPError as e: status = f"Submission Failed: {e.response.status_code} {e.response.text[:500]}" except Exception as e: # noqa: BLE001 status = f"Submission Failed: {e}" df = pd.DataFrame([{"Task ID": t, "Submitted Answer": a} for t, a in cache.items()]) return status, df def clear_cache(): if CACHE_FILE.exists(): CACHE_FILE.unlink() return "Cache cleared." with gr.Blocks() as demo: gr.Markdown("# GAIA Level-1 Agent – Evaluation Runner") gr.Markdown( """ 1. Log in with your Hugging Face account. 2. **Run agent** answers all 20 questions (takes several minutes). Answers are cached per question, so re-running only fills in what is missing. 3. **Submit answers** sends the cached answers for scoring and updates the leaderboard. Model is chosen with env vars: `MODEL_ID` (LiteLLM id + provider API key) or `HF_MODEL_ID` + `HF_TOKEN`. See `agent.py`. """ ) gr.LoginButton() with gr.Row(): use_cache = gr.Checkbox(value=True, label="Reuse cached answers") run_btn = gr.Button("Run agent", variant="primary") submit_btn = gr.Button("Submit answers") clear_btn = gr.Button("Clear cache") status = gr.Textbox(label="Status", lines=5, interactive=False) table = gr.DataFrame(label="Questions and answers", wrap=True) run_btn.click(run_agent, inputs=[use_cache], outputs=[status, table]) submit_btn.click(submit_answers, outputs=[status, table]) clear_btn.click(clear_cache, outputs=[status]) if __name__ == "__main__": demo.launch(debug=True, share=False)