| """Public Space UI: reviewable dry-runs and a separate explicit submission.""" |
|
|
| from __future__ import annotations |
|
|
| import json |
| import os |
| from pathlib import Path |
| from typing import Any |
|
|
| import gradio as gr |
| import pandas as pd |
| from huggingface_hub import get_token |
|
|
| from agent import GaiaAgent |
| from agent_code import resolve_agent_code |
| from cache import AnswerCache, ResultStore |
| from config import Settings |
| from evaluation import run_evaluation, submission_answers |
| from gaia_client import GaiaClient |
|
|
| SETTINGS = Settings.from_env() |
| PROJECT_ROOT = Path(__file__).resolve().parent |
| RESULT_STORE = ResultStore(Path(os.getenv("GAIA_RESULTS_PATH", "results/results.json"))) |
|
|
|
|
| def has_huggingface_login() -> bool: |
| """Detect Space OAuth or local CLI auth without retaining or logging the token.""" |
| if os.getenv("SPACE_ID") or os.getenv("HF_TOKEN"): |
| return True |
| try: |
| return bool(get_token()) |
| except OSError: |
| return False |
|
|
|
|
| def _frame(results: list[dict[str, Any]]) -> pd.DataFrame: |
| return pd.DataFrame( |
| [ |
| { |
| "Task ID": row.get("task_id", ""), |
| "Task Type": row.get("task_type", ""), |
| "Question": row.get("question", ""), |
| "Generated Answer": row.get("answer", ""), |
| "Seconds": row.get("duration_seconds", 0), |
| "Status": row.get("status", ""), |
| "Error": row.get("error", ""), |
| "Evidence": json.dumps(row.get("evidence", []), ensure_ascii=False), |
| } |
| for row in results |
| ] |
| ) |
|
|
|
|
| def _ordered_all(client: GaiaClient) -> list[dict[str, Any]]: |
| ids = [str(task["task_id"]) for task in client.get_questions()] |
| return RESULT_STORE.ordered(ids) |
|
|
|
|
| def run_dry_evaluation(force: bool = False) -> tuple[str, pd.DataFrame]: |
| """Answer every task and checkpoint results; this function cannot submit.""" |
| try: |
| client = GaiaClient(SETTINGS) |
| results = run_evaluation( |
| client, |
| GaiaAgent(SETTINGS), |
| RESULT_STORE, |
| force=bool(force), |
| ) |
| completed = sum(row["status"] == "ok" for row in results) |
| failed = len(results) - completed |
| status = ( |
| f"Dry run complete: {completed}/{len(results)} answered; {failed} failed. " |
| "No answers were submitted." |
| ) |
| if len(results) == 20 and failed == 0: |
| status += " All 20 unique tasks are ready for explicit submission." |
| return status, _frame(results) |
| except Exception as exc: |
| return f"Dry run failed: {type(exc).__name__}: {exc}", _frame( |
| RESULT_STORE.ordered() |
| ) |
|
|
|
|
| def rerun_task(task_id: str) -> tuple[str, pd.DataFrame]: |
| """Force one task to run again while retaining other checkpoints.""" |
| task_id = str(task_id or "").strip() |
| if not task_id: |
| return "Enter a Task ID to rerun.", _frame(RESULT_STORE.ordered()) |
| try: |
| client = GaiaClient(SETTINGS) |
| run_evaluation( |
| client, |
| GaiaAgent( |
| SETTINGS, |
| AnswerCache(SETTINGS.cache_dir / "answers.json", enabled=False), |
| ), |
| RESULT_STORE, |
| force=True, |
| task_ids={task_id}, |
| ) |
| results = _ordered_all(client) |
| row = RESULT_STORE.load()[task_id] |
| return f"Rerun {task_id}: {row['status']}", _frame(results) |
| except Exception as exc: |
| return f"Rerun failed: {type(exc).__name__}: {exc}", _frame( |
| RESULT_STORE.ordered() |
| ) |
|
|
|
|
| def submit_cached_answers(profile: gr.OAuthProfile | None) -> tuple[str, pd.DataFrame]: |
| """The sole explicit route to the official submission POST.""" |
| try: |
| client = GaiaClient(SETTINGS) |
| results = _ordered_all(client) |
| except Exception as exc: |
| return f"Could not validate cached run: {type(exc).__name__}: {exc}", _frame( |
| RESULT_STORE.ordered() |
| ) |
| if profile is None: |
| return "Please log in to Hugging Face before submitting.", _frame(results) |
| try: |
| answers = submission_answers(results, expected_count=20) |
| agent_code = resolve_agent_code( |
| space_id=os.getenv("SPACE_ID"), |
| configured_url=SETTINGS.agent_code_url, |
| allow_inline=SETTINGS.allow_inline_agent_code, |
| root=PROJECT_ROOT, |
| ) |
| response = client.submit_answers( |
| username=str(profile.username).strip(), |
| agent_code=agent_code, |
| answers=answers, |
| ) |
| status = ( |
| "Submission successful.\n" |
| f"User: {response.get('username', profile.username)}\n" |
| f"Overall Score: {response.get('score', 'N/A')}% " |
| f"({response.get('correct_count', '?')}/{response.get('total_attempted', '?')} correct)\n" |
| f"Message: {response.get('message', 'No message received.')}" |
| ) |
| return status, _frame(results) |
| except Exception as exc: |
| return f"Submission not sent: {type(exc).__name__}: {exc}", _frame(results) |
|
|
|
|
| with gr.Blocks() as demo: |
| gr.Markdown("# Modular GAIA Level-1 Agent") |
| gr.Markdown( |
| "Run or resume a dry evaluation, review all answers/evidence, and rerun individual " |
| "tasks. Submission is a distinct action and is enabled logically only when exactly 20 " |
| "unique tasks have non-empty successful answers. Keep this Space public." |
| ) |
| |
| |
| if has_huggingface_login(): |
| gr.LoginButton() |
| else: |
| gr.Markdown( |
| "Local mode: run `hf auth login`, then expose `HF_TOKEN` to this process for " |
| "inference. Configure `GAIA_AGENT_CODE_URL`, or explicitly enable inline source." |
| ) |
| with gr.Row(): |
| force_all = gr.Checkbox(label="Force rerun all tasks", value=False) |
| run_button = gr.Button("Run / Resume Dry Evaluation", variant="primary") |
| submit_button = gr.Button("Submit Reviewed Complete Run", variant="secondary") |
| with gr.Row(): |
| rerun_id = gr.Textbox(label="Task ID to rerun") |
| rerun_button = gr.Button("Rerun Selected Task") |
| status_output = gr.Textbox(label="Status", lines=5, interactive=False) |
| results_table = gr.DataFrame(label="20-question review", wrap=True) |
| run_button.click( |
| run_dry_evaluation, inputs=[force_all], outputs=[status_output, results_table] |
| ) |
| rerun_button.click( |
| rerun_task, inputs=[rerun_id], outputs=[status_output, results_table] |
| ) |
| submit_button.click(submit_cached_answers, outputs=[status_output, results_table]) |
|
|
|
|
| if __name__ == "__main__": |
| demo.queue(default_concurrency_limit=1).launch(debug=False, share=False) |
|
|
