import json import os import time from pathlib import Path from tempfile import TemporaryDirectory from urllib.parse import quote import gradio as gr import pandas as pd import requests try: from dotenv import load_dotenv except ImportError: load_dotenv = None try: import spaces except ImportError: spaces = None from agent import BasicAgent if load_dotenv: load_dotenv() DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" CACHE_PATH = Path("answers_cache.json") def log(message: str) -> None: print(f"[{time.strftime('%H:%M:%S')}] {message}", flush=True) if spaces: @spaces.GPU(duration=1) def dummy_gpu(): return None def load_cache() -> dict: if not CACHE_PATH.exists(): return {} try: return json.loads(CACHE_PATH.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return {} def save_cache(cache: dict) -> None: CACHE_PATH.write_text(json.dumps(cache, indent=2, ensure_ascii=False), encoding="utf-8") def fetch_questions(api_url: str) -> list[dict]: questions_url = f"{api_url}/questions" log(f"Fetching questions from {questions_url}") response = requests.get(questions_url, timeout=30) response.raise_for_status() questions = response.json() if not isinstance(questions, list) or not questions: raise ValueError("Questions endpoint returned an empty or invalid payload.") return questions def download_attachment(item: dict, api_url: str, temp_dir: str) -> tuple[str | None, str | None]: file_name = item.get("file_name") task_id = item.get("task_id") if not file_name or not task_id: return None, None safe_name = Path(str(file_name)).name encoded_task_id = quote(str(task_id), safe="") attachment_path = Path(temp_dir) / f"{encoded_task_id}_{safe_name}" file_url = f"{api_url}/files/{encoded_task_id}" try: log(f"Downloading attachment for task {task_id}: {safe_name}") response = requests.get(file_url, timeout=60) response.raise_for_status() except requests.exceptions.HTTPError as exc: if exc.response is not None and exc.response.status_code == 404: log(f"Attachment 404 for task {task_id}: {safe_name}") return None, f"Attachment listed as {safe_name}, but the scoring API returned 404 for {file_url}." raise attachment_path.write_bytes(response.content) log(f"Attachment saved for task {task_id}: {attachment_path}") return str(attachment_path), None def rows_to_dataframe(rows: list[dict]) -> pd.DataFrame: return pd.DataFrame( rows, columns=["Task ID", "Question", "File", "Submitted Answer", "Source", "Status"], ) def cache_to_payload(cache: dict) -> list[dict]: payload = [] for task_id, item in cache.items(): submitted_answer = item.get("submitted_answer") if submitted_answer and not str(submitted_answer).startswith("AGENT ERROR:"): payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) return payload def run_agent(profile: gr.OAuthProfile | None, run_all_questions: bool, use_cache: bool): if not profile: return "Please log in to Hugging Face first.", [], rows_to_dataframe([]) api_url = DEFAULT_API_URL cache = load_cache() answers_payload = [] results_log = [] try: questions_data = fetch_questions(api_url) except Exception as exc: return f"Error fetching questions: {exc}", [], rows_to_dataframe([]) if not run_all_questions: questions_data = questions_data[:1] try: agent = BasicAgent() except Exception as exc: return f"Error initializing agent: {exc}", [], rows_to_dataframe([]) with TemporaryDirectory() as temp_dir: for index, item in enumerate(questions_data, start=1): task_id = item.get("task_id") question_text = item.get("question") file_name = item.get("file_name") or "" if not task_id or question_text is None: results_log.append( { "Task ID": task_id or "missing", "Question": question_text or "", "File": file_name, "Submitted Answer": "", "Source": "skip", "Status": "Missing task_id or question.", } ) continue cache_key = str(task_id) log(f"Task {index}/{len(questions_data)} started: {task_id}") if use_cache and cache_key in cache: submitted_answer = cache[cache_key]["submitted_answer"] source = "cache" status = "Loaded from answers_cache.json." log(f"Task {index}/{len(questions_data)} cache hit: {task_id} -> {submitted_answer}") else: try: attachment_path, attachment_error = download_attachment(item, api_url, temp_dir) question_for_agent = question_text if attachment_error: question_for_agent += f"\n\n{attachment_error}" submitted_answer = agent( question=question_for_agent, task_id=str(task_id), file_path=attachment_path, file_name=file_name or None, ) source = "agent" status = "Generated." cache[cache_key] = { "question": question_text, "file_name": file_name, "submitted_answer": submitted_answer, } save_cache(cache) log(f"Task {index}/{len(questions_data)} cached answer: {task_id}") except Exception as exc: submitted_answer = f"AGENT ERROR: {exc}" source = "error" status = str(exc) log(f"Task {index}/{len(questions_data)} error: {task_id}: {exc}") answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) results_log.append( { "Task ID": task_id, "Question": question_text, "File": file_name, "Submitted Answer": submitted_answer, "Source": source, "Status": status, } ) log(f"[{index}/{len(questions_data)}] {task_id}: {submitted_answer}") status = ( f"Generated {len(answers_payload)} answer(s) for {profile.username}. " "Review the table, then submit when ready." ) return status, answers_payload, rows_to_dataframe(results_log) def submit_answers(profile: gr.OAuthProfile | None, answers_payload: list[dict] | None): if not profile: return "Please log in to Hugging Face first." source_note = "current run" if not answers_payload: answers_payload = cache_to_payload(load_cache()) if not answers_payload: return "No answers to submit. Run the agent first." source_note = "answers_cache.json" space_id = os.getenv("SPACE_ID") if space_id: agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" else: agent_code = f"https://huggingface.co/spaces/{profile.username}/YOUR_SPACE_NAME/tree/main" submission_data = { "username": profile.username.strip(), "agent_code": agent_code, "answers": answers_payload, } try: response = requests.post(f"{DEFAULT_API_URL}/submit", json=submission_data, timeout=90) response.raise_for_status() result_data = response.json() return ( "Submission Successful!\n" f"User: {result_data.get('username')}\n" f"Submitted: {len(answers_payload)} answer(s) from {source_note}\n" f"Overall Score: {result_data.get('score', 'N/A')}% " f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" f"Message: {result_data.get('message', 'No message received.')}\n" f"Code: {agent_code}" ) except requests.exceptions.HTTPError as exc: detail = exc.response.text try: detail = exc.response.json().get("detail", detail) except requests.exceptions.JSONDecodeError: pass return f"Submission failed: {exc.response.status_code}. {detail}" except Exception as exc: return f"Submission failed: {exc}" def run_and_submit_all(profile: gr.OAuthProfile | None): status, answers_payload, results_df = run_agent( profile=profile, run_all_questions=True, use_cache=True, ) if not answers_payload: return status, results_df submit_status = submit_answers(profile, answers_payload) return f"{status}\n\n{submit_status}", results_df with gr.Blocks() as demo: gr.Markdown("# Basic Agent Evaluation Runner") gr.Markdown( """ **Instructions:** 1. Log in to your Hugging Face account using the button below. 2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run the agent, submit answers, and see the score. --- **Disclaimers:** This can take quite some time while the agent works through all questions. """ ) gr.LoginButton() run_button = gr.Button("Run Evaluation & Submit All Answers") status_output = gr.Textbox(label="Run Status / Submission Result", lines=8, interactive=False) results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) run_button.click( fn=run_and_submit_all, outputs=[status_output, results_table], ) if __name__ == "__main__": print("\n" + "-" * 30 + " App Starting " + "-" * 30) space_host_startup = os.getenv("SPACE_HOST") space_id_startup = os.getenv("SPACE_ID") if space_host_startup: print(f"SPACE_HOST found: {space_host_startup}") print(f"Runtime URL should be: https://{space_host_startup}.hf.space") else: print("SPACE_HOST environment variable not found (running locally?).") if space_id_startup: print(f"SPACE_ID found: {space_id_startup}") print(f"Repo URL: https://huggingface.co/spaces/{space_id_startup}") else: print("SPACE_ID environment variable not found. Repo URL cannot be determined.") print("-" * 74 + "\n") demo.launch(debug=True, share=False)