import os import gradio as gr import spaces import text import orchestration import results_view # --- Constants --- DEFAULT_API_URL = os.getenv("GAIA_API_URL", "https://agents-course-unit4-scoring.hf.space") # ZeroGPU requires at least one @spaces.GPU-decorated function to detect at # startup (docs/adr/0002 keeps the Space text/web-only, so the GPU is not used # for inference here — the decorator just satisfies the ZeroGPU runtime, and is # a no-op when running locally or on CPU hardware). @spaces.GPU def run_and_submit_all(profile: gr.OAuthProfile | None): """ Fetches the served Questions, submits the committed Answer bundle for every Task it recognizes (without calling the Agent), runs the live Agent (best-effort, text/web-only) for Tasks missing from the bundle — downloading their Attachments from the server first — posts the submission payload, and returns the Benchmark results. Returns ``(status_message, results_html)``: the first element is a concise run/submission status; the second is the rendered Benchmark results view (score card + per-Question table + expandable per-Task Worklog — ticket 05) as an HTML fragment for a ``gr.HTML`` component. """ # --- Determine HF Space Repo URL --- space_id = os.getenv("SPACE_ID") if profile: username = f"{profile.username}" print(f"User logged in: {username}") else: print("User not logged in.") return "Please Login to Hugging Face with the button.", None # In the case of an app running as a Hugging Face space, this link points # toward your codebase (useful for others so please keep it public) agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "" print(agent_code) # Thin client for the scoring server (fetch /questions, download /files/{id}, # POST /submit) server = orchestration.ServerAPI(DEFAULT_API_URL) # 1. Fetch Questions print(f"Fetching questions from: {DEFAULT_API_URL}/questions") try: questions = server.fetch_questions() if not questions: print("Fetched questions list is empty.") return "Fetched questions list is empty or invalid format.", None print(f"Fetched {len(questions)} questions.") except Exception as e: print(f"Error fetching questions: {e}") return f"Error fetching questions: {e}", None # 2. Load the committed Answer bundle (generated by local Runs, keyed by # task_id). Bundled Answers are submitted without re-solving (ADR-0001). bundle = text._load_answer_bundle() print(f"Answer bundle holds {len(bundle)} Task(s).") # 3. Build the live Agent lazily — only needed when Tasks are missing from # the bundle, so a fully-bundled Run needs no DEEPSEEK_API_KEY. The Space # fallback is text/web-only (ADR-0002): no local vision/audio models load. unbundled = [ t for t in questions if t.get("task_id") and t.get("task_id") not in bundle ] agent = None if unbundled: try: agent = text.GAIASolverAgent(multimodal=False) print(f"Built live Agent for {len(unbundled)} unbundled Task(s).") except Exception as e: print(f"⚠️ Could not build the live Agent (best-effort): {e}") # 4. Orchestrate: build the submission payload + results view (bundle match # → bundled Answer/Worklog; unbundled → live solve, downloading the # Attachment from the server first) try: payload, view = orchestration.orchestrate_run( questions, bundle, agent, server, username=username, agent_code=agent_code, ) except Exception as e: print(f"Error orchestrating the run: {e}") return f"Error orchestrating the run: {e}", None if not payload["answers"]: print("No answers to submit.") return "No answers to submit.", None # 5. Submit the payload and capture the Benchmark results (ScoreResponse) bundled_count = sum(1 for r in view["rows"] if r["source"] == "bundle") live_count = sum(1 for r in view["rows"] if r["source"] == "live") status_update = ( f"Submitting {len(payload['answers'])} answers for user '{username}' " f"({bundled_count} from bundle, {live_count} live)..." ) print(status_update) try: score_response = server.submit(payload) except Exception as e: print(f"Submission failed: {e}") return f"Submission Failed: {e}", results_view.render_results_view(view) view["score_card"].update(orchestration.parse_score_response(score_response)) # 6. Build the concise status message + the rich results view (score card, # per-Question table, expandable per-Task Worklog — ticket 05). sc = view["score_card"] score = sc.get("score") score_display = "N/A" if score is None else f"{score}%" correct = sc.get("correct_count") total = sc.get("total_attempted") correct_display = "?" if correct is None else correct total_display = "?" if total is None else total final_status = ( "Submission Successful!\n" f"User: {sc.get('username')}\n" f"Overall Score: {score_display} ({correct_display}/{total_display} correct)\n" f"Message: {sc.get('message', 'No message received.')}" ) submitted_at = sc.get("timestamp") if submitted_at: final_status += f"\nSubmitted at: {submitted_at}" print("Submission successful.") return final_status, results_view.render_results_view(view) # --- Build Gradio Interface using Blocks --- with gr.Blocks() as demo: gr.Markdown("# GAIA Solver Evaluation Runner") gr.Markdown( """ **Instructions:** 1. Log in to your Hugging Face account using the button below. This uses your HF username for submission. 2. Click 'Run Evaluation & Submit All Answers' to submit the pre-computed Answer bundle, run the Agent live (best-effort) for any Questions missing from it, and see the Benchmark results. --- **How it works:** Answers are pre-computed locally (see `text.py`) into a committed Answer bundle (`answer_bundle.json`). This Space submits those answers directly for every Question it recognizes — no re-solving — and only runs the live Agent (text/web-only) for Questions missing from the bundle. """ ) gr.LoginButton() run_button = gr.Button("Run Evaluation & Submit All Answers") status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) # Ticket 05: the rich Benchmark results view — score card, per-Question # table of submitted Answers (Level + source), and an expandable per-Task # Worklog (tool summary + full trace) — rendered as one HTML fragment. results_output = gr.HTML(label="Benchmark Results") run_button.click( fn=run_and_submit_all, outputs=[status_output, results_output] ) if __name__ == "__main__": print("\n" + "-"*30 + " App Starting " + "-"*30) # Check for SPACE_HOST and SPACE_ID at startup for information space_host_startup = os.getenv("SPACE_HOST") space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup if space_host_startup: print(f"✅ SPACE_HOST found: {space_host_startup}") print(f" Runtime URL should be: https://{space_host_startup}.hf.space") else: print("ℹ️ SPACE_HOST environment variable not found (running locally?).") if space_id_startup: # Print repo URLs if SPACE_ID is found print(f"✅ SPACE_ID found: {space_id_startup}") print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}") print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main") else: print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") print("-"*(60 + len(" App Starting ")) + "\n") print("Launching Gradio Interface for GAIA Solver Evaluation...") demo.launch(debug=True, share=False)