Final_Assignment_Template

Sleeping

Add @spaces.GPU decorator for ZeroGPU hardware

f874229 3 months ago

8.17 kB

import os
import gradio as gr
import spaces

import text
import orchestration
import results_view

# --- Constants ---
DEFAULT_API_URL = os.getenv("GAIA_API_URL", "https://agents-course-unit4-scoring.hf.space")


# ZeroGPU requires at least one @spaces.GPU-decorated function to detect at
# startup (docs/adr/0002 keeps the Space text/web-only, so the GPU is not used
# for inference here — the decorator just satisfies the ZeroGPU runtime, and is
# a no-op when running locally or on CPU hardware).
@spaces.GPU
def run_and_submit_all(profile: gr.OAuthProfile | None):
    """
    Fetches the served Questions, submits the committed Answer bundle for every
    Task it recognizes (without calling the Agent), runs the live Agent
    (best-effort, text/web-only) for Tasks missing from the bundle — downloading
    their Attachments from the server first — posts the submission payload, and
    returns the Benchmark results.
    Returns ``(status_message, results_html)``: the first element is a concise
    run/submission status; the second is the rendered Benchmark results view
    (score card + per-Question table + expandable per-Task Worklog — ticket 05)
    as an HTML fragment for a ``gr.HTML`` component.
    """
    # --- Determine HF Space Repo URL ---
    space_id = os.getenv("SPACE_ID")

    if profile:
        username = f"{profile.username}"
        print(f"User logged in: {username}")
    else:
        print("User not logged in.")
        return "Please Login to Hugging Face with the button.", None

    # In the case of an app running as a Hugging Face space, this link points
    # toward your codebase (useful for others so please keep it public)
    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else ""
    print(agent_code)

    # Thin client for the scoring server (fetch /questions, download /files/{id},
    # POST /submit)
    server = orchestration.ServerAPI(DEFAULT_API_URL)

    # 1. Fetch Questions
    print(f"Fetching questions from: {DEFAULT_API_URL}/questions")
    try:
        questions = server.fetch_questions()
        if not questions:
            print("Fetched questions list is empty.")
            return "Fetched questions list is empty or invalid format.", None
        print(f"Fetched {len(questions)} questions.")
    except Exception as e:
        print(f"Error fetching questions: {e}")
        return f"Error fetching questions: {e}", None

    # 2. Load the committed Answer bundle (generated by local Runs, keyed by
    # task_id). Bundled Answers are submitted without re-solving (ADR-0001).
    bundle = text._load_answer_bundle()
    print(f"Answer bundle holds {len(bundle)} Task(s).")

    # 3. Build the live Agent lazily — only needed when Tasks are missing from
    # the bundle, so a fully-bundled Run needs no DEEPSEEK_API_KEY. The Space
    # fallback is text/web-only (ADR-0002): no local vision/audio models load.
    unbundled = [
        t for t in questions
        if t.get("task_id") and t.get("task_id") not in bundle
    ]
    agent = None
    if unbundled:
        try:
            agent = text.GAIASolverAgent(multimodal=False)
            print(f"Built live Agent for {len(unbundled)} unbundled Task(s).")
        except Exception as e:
            print(f"⚠️ Could not build the live Agent (best-effort): {e}")

    # 4. Orchestrate: build the submission payload + results view (bundle match
    # → bundled Answer/Worklog; unbundled → live solve, downloading the
    # Attachment from the server first)
    try:
        payload, view = orchestration.orchestrate_run(
            questions, bundle, agent, server,
            username=username, agent_code=agent_code,
        )
    except Exception as e:
        print(f"Error orchestrating the run: {e}")
        return f"Error orchestrating the run: {e}", None

    if not payload["answers"]:
        print("No answers to submit.")
        return "No answers to submit.", None

    # 5. Submit the payload and capture the Benchmark results (ScoreResponse)
    bundled_count = sum(1 for r in view["rows"] if r["source"] == "bundle")
    live_count = sum(1 for r in view["rows"] if r["source"] == "live")
    status_update = (
        f"Submitting {len(payload['answers'])} answers for user '{username}' "
        f"({bundled_count} from bundle, {live_count} live)..."
    )
    print(status_update)
    try:
        score_response = server.submit(payload)
    except Exception as e:
        print(f"Submission failed: {e}")
        return f"Submission Failed: {e}", results_view.render_results_view(view)

    view["score_card"].update(orchestration.parse_score_response(score_response))

    # 6. Build the concise status message + the rich results view (score card,
    # per-Question table, expandable per-Task Worklog — ticket 05).
    sc = view["score_card"]
    score = sc.get("score")
    score_display = "N/A" if score is None else f"{score}%"
    correct = sc.get("correct_count")
    total = sc.get("total_attempted")
    correct_display = "?" if correct is None else correct
    total_display = "?" if total is None else total
    final_status = (
        "Submission Successful!\n"
        f"User: {sc.get('username')}\n"
        f"Overall Score: {score_display} ({correct_display}/{total_display} correct)\n"
        f"Message: {sc.get('message', 'No message received.')}"
    )
    submitted_at = sc.get("timestamp")
    if submitted_at:
        final_status += f"\nSubmitted at: {submitted_at}"
    print("Submission successful.")
    return final_status, results_view.render_results_view(view)


# --- Build Gradio Interface using Blocks ---
with gr.Blocks() as demo:
    gr.Markdown("# GAIA Solver Evaluation Runner")
    gr.Markdown(
        """
        **Instructions:**
        1.  Log in to your Hugging Face account using the button below. This uses your HF username for submission.
        2.  Click 'Run Evaluation & Submit All Answers' to submit the pre-computed Answer bundle, run the Agent live (best-effort) for any Questions missing from it, and see the Benchmark results.
        ---
        **How it works:**
        Answers are pre-computed locally (see `text.py`) into a committed Answer
        bundle (`answer_bundle.json`). This Space submits those answers directly
        for every Question it recognizes — no re-solving — and only runs the live
        Agent (text/web-only) for Questions missing from the bundle.
        """
    )

    gr.LoginButton()

    run_button = gr.Button("Run Evaluation & Submit All Answers")

    status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
    # Ticket 05: the rich Benchmark results view — score card, per-Question
    # table of submitted Answers (Level + source), and an expandable per-Task
    # Worklog (tool summary + full trace) — rendered as one HTML fragment.
    results_output = gr.HTML(label="Benchmark Results")

    run_button.click(
        fn=run_and_submit_all,
        outputs=[status_output, results_output]
    )

if __name__ == "__main__":
    print("\n" + "-"*30 + " App Starting " + "-"*30)
    # Check for SPACE_HOST and SPACE_ID at startup for information
    space_host_startup = os.getenv("SPACE_HOST")
    space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup

    if space_host_startup:
        print(f"✅ SPACE_HOST found: {space_host_startup}")
        print(f"   Runtime URL should be: https://{space_host_startup}.hf.space")
    else:
        print("ℹ️  SPACE_HOST environment variable not found (running locally?).")

    if space_id_startup: # Print repo URLs if SPACE_ID is found
        print(f"✅ SPACE_ID found: {space_id_startup}")
        print(f"   Repo URL: https://huggingface.co/spaces/{space_id_startup}")
        print(f"   Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
    else:
        print("ℹ️  SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")

    print("-"*(60 + len(" App Starting ")) + "\n")

    print("Launching Gradio Interface for GAIA Solver Evaluation...")
    demo.launch(debug=True, share=False)

	import os
	import gradio as gr
	import spaces

	import text
	import orchestration
	import results_view

	# --- Constants ---
	DEFAULT_API_URL = os.getenv("GAIA_API_URL", "https://agents-course-unit4-scoring.hf.space")


	# ZeroGPU requires at least one @spaces.GPU-decorated function to detect at
	# startup (docs/adr/0002 keeps the Space text/web-only, so the GPU is not used
	# for inference here — the decorator just satisfies the ZeroGPU runtime, and is
	# a no-op when running locally or on CPU hardware).
	@spaces.GPU
	def run_and_submit_all(profile: gr.OAuthProfile \| None):
	"""
	Fetches the served Questions, submits the committed Answer bundle for every
	Task it recognizes (without calling the Agent), runs the live Agent
	(best-effort, text/web-only) for Tasks missing from the bundle — downloading
	their Attachments from the server first — posts the submission payload, and
	returns the Benchmark results.

	Returns ``(status_message, results_html)``: the first element is a concise
	run/submission status; the second is the rendered Benchmark results view
	(score card + per-Question table + expandable per-Task Worklog — ticket 05)
	as an HTML fragment for a ``gr.HTML`` component.
	"""
	# --- Determine HF Space Repo URL ---
	space_id = os.getenv("SPACE_ID")

	if profile:
	username = f"{profile.username}"
	print(f"User logged in: {username}")
	else:
	print("User not logged in.")
	return "Please Login to Hugging Face with the button.", None

	# In the case of an app running as a Hugging Face space, this link points
	# toward your codebase (useful for others so please keep it public)
	agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else ""
	print(agent_code)

	# Thin client for the scoring server (fetch /questions, download /files/{id},
	# POST /submit)
	server = orchestration.ServerAPI(DEFAULT_API_URL)

	# 1. Fetch Questions
	print(f"Fetching questions from: {DEFAULT_API_URL}/questions")
	try:
	questions = server.fetch_questions()
	if not questions:
	print("Fetched questions list is empty.")
	return "Fetched questions list is empty or invalid format.", None
	print(f"Fetched {len(questions)} questions.")
	except Exception as e:
	print(f"Error fetching questions: {e}")
	return f"Error fetching questions: {e}", None

	# 2. Load the committed Answer bundle (generated by local Runs, keyed by
	# task_id). Bundled Answers are submitted without re-solving (ADR-0001).
	bundle = text._load_answer_bundle()
	print(f"Answer bundle holds {len(bundle)} Task(s).")

	# 3. Build the live Agent lazily — only needed when Tasks are missing from
	# the bundle, so a fully-bundled Run needs no DEEPSEEK_API_KEY. The Space
	# fallback is text/web-only (ADR-0002): no local vision/audio models load.
	unbundled = [
	t for t in questions
	if t.get("task_id") and t.get("task_id") not in bundle
	]
	agent = None
	if unbundled:
	try:
	agent = text.GAIASolverAgent(multimodal=False)
	print(f"Built live Agent for {len(unbundled)} unbundled Task(s).")
	except Exception as e:
	print(f"⚠️ Could not build the live Agent (best-effort): {e}")

	# 4. Orchestrate: build the submission payload + results view (bundle match
	# → bundled Answer/Worklog; unbundled → live solve, downloading the
	# Attachment from the server first)
	try:
	payload, view = orchestration.orchestrate_run(
	questions, bundle, agent, server,
	username=username, agent_code=agent_code,
	)
	except Exception as e:
	print(f"Error orchestrating the run: {e}")
	return f"Error orchestrating the run: {e}", None

	if not payload["answers"]:
	print("No answers to submit.")
	return "No answers to submit.", None

	# 5. Submit the payload and capture the Benchmark results (ScoreResponse)
	bundled_count = sum(1 for r in view["rows"] if r["source"] == "bundle")
	live_count = sum(1 for r in view["rows"] if r["source"] == "live")
	status_update = (
	f"Submitting {len(payload['answers'])} answers for user '{username}' "
	f"({bundled_count} from bundle, {live_count} live)..."
	)
	print(status_update)
	try:
	score_response = server.submit(payload)
	except Exception as e:
	print(f"Submission failed: {e}")
	return f"Submission Failed: {e}", results_view.render_results_view(view)

	view["score_card"].update(orchestration.parse_score_response(score_response))

	# 6. Build the concise status message + the rich results view (score card,
	# per-Question table, expandable per-Task Worklog — ticket 05).
	sc = view["score_card"]
	score = sc.get("score")
	score_display = "N/A" if score is None else f"{score}%"
	correct = sc.get("correct_count")
	total = sc.get("total_attempted")
	correct_display = "?" if correct is None else correct
	total_display = "?" if total is None else total
	final_status = (
	"Submission Successful!\n"
	f"User: {sc.get('username')}\n"
	f"Overall Score: {score_display} ({correct_display}/{total_display} correct)\n"
	f"Message: {sc.get('message', 'No message received.')}"
	)
	submitted_at = sc.get("timestamp")
	if submitted_at:
	final_status += f"\nSubmitted at: {submitted_at}"
	print("Submission successful.")
	return final_status, results_view.render_results_view(view)


	# --- Build Gradio Interface using Blocks ---
	with gr.Blocks() as demo:
	gr.Markdown("# GAIA Solver Evaluation Runner")
	gr.Markdown(
	"""
	Instructions:

	1. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
	2. Click 'Run Evaluation & Submit All Answers' to submit the pre-computed Answer bundle, run the Agent live (best-effort) for any Questions missing from it, and see the Benchmark results.

	---
	How it works:
	Answers are pre-computed locally (see `text.py`) into a committed Answer
	bundle (`answer_bundle.json`). This Space submits those answers directly
	for every Question it recognizes — no re-solving — and only runs the live
	Agent (text/web-only) for Questions missing from the bundle.
	"""
	)

	gr.LoginButton()

	run_button = gr.Button("Run Evaluation & Submit All Answers")

	status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False)
	# Ticket 05: the rich Benchmark results view — score card, per-Question
	# table of submitted Answers (Level + source), and an expandable per-Task
	# Worklog (tool summary + full trace) — rendered as one HTML fragment.
	results_output = gr.HTML(label="Benchmark Results")

	run_button.click(
	fn=run_and_submit_all,
	outputs=[status_output, results_output]
	)

	if __name__ == "__main__":
	print("\n" + "-"30 + " App Starting " + "-"30)
	# Check for SPACE_HOST and SPACE_ID at startup for information
	space_host_startup = os.getenv("SPACE_HOST")
	space_id_startup = os.getenv("SPACE_ID") # Get SPACE_ID at startup

	if space_host_startup:
	print(f"✅ SPACE_HOST found: {space_host_startup}")
	print(f" Runtime URL should be: https://{space_host_startup}.hf.space")
	else:
	print("ℹ️ SPACE_HOST environment variable not found (running locally?).")

	if space_id_startup: # Print repo URLs if SPACE_ID is found
	print(f"✅ SPACE_ID found: {space_id_startup}")
	print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}")
	print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main")
	else:
	print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.")

	print("-"*(60 + len(" App Starting ")) + "\n")

	print("Launching Gradio Interface for GAIA Solver Evaluation...")
	demo.launch(debug=True, share=False)