import os import gradio as gr import requests import pandas as pd from smolagents import CodeAgent, DuckDuckGoSearchTool, HfApiModel # --- Constants --- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" # --- Agent Definition --- class BasicAgent: def __init__(self): print("Initializing agent...") self.model = HfApiModel( model_id="Qwen/Qwen2.5-Coder-32B-Instruct" ) self.agent = CodeAgent( tools=[ DuckDuckGoSearchTool() ], model=self.model, max_steps=10 ) print("Agent initialized successfully.") def __call__(self, question: str) -> str: print(f"Agent received question: {question[:150]}...") prompt = f""" You are an expert research agent answering a benchmark question. Solve the question carefully and accurately. IMPORTANT INSTRUCTIONS: 1. Analyze exactly what the question is asking. 2. Use web search whenever external information is needed. 3. Prefer reliable sources and cross-check important facts. 4. If the question refers to Wikipedia, search for the relevant Wikipedia information. 5. If the question contains a URL, investigate the URL when possible. 6. If the question requires calculations, use Python/code when useful. 7. If the question asks for a specific format, follow that format exactly. 8. Do not guess when you can research the answer. 9. Return ONLY the final answer. 10. Do not include your reasoning or analysis. 11. Do not say "Here is the answer". 12. Do not add unnecessary explanation. Question: {question} """ try: result = self.agent.run(prompt) answer = str(result).strip() print(f"Agent answer: {answer}") return answer except Exception as e: print(f"Agent error: {e}") return f"Unable to answer: {e}" # --- Evaluation and Submission --- def run_and_submit_all(profile: gr.OAuthProfile | None): """ Fetch all benchmark questions, run the agent, submit the answers, and display the results. """ space_id = os.getenv("SPACE_ID") if profile: username = f"{profile.username}" print(f"User logged in: {username}") else: return "Please Login to Hugging Face with the button.", None api_url = DEFAULT_API_URL questions_url = f"{api_url}/questions" submit_url = f"{api_url}/submit" # --- Initialize Agent --- try: agent = BasicAgent() except Exception as e: print(f"Error initializing agent: {e}") return f"Error initializing agent: {e}", None # --- Link to the Space code --- agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" # --- Fetch Questions --- print(f"Fetching questions from: {questions_url}") try: response = requests.get( questions_url, timeout=30 ) response.raise_for_status() questions_data = response.json() if not questions_data: return "No questions were returned.", None print(f"Fetched {len(questions_data)} questions.") except requests.exceptions.RequestException as e: return f"Error fetching questions: {e}", None except Exception as e: return f"Unexpected error fetching questions: {e}", None # --- Run Agent on Questions --- results_log = [] answers_payload = [] print(f"Running agent on {len(questions_data)} questions...") for index, item in enumerate(questions_data, start=1): task_id = item.get("task_id") question_text = item.get("question") if not task_id or question_text is None: print(f"Skipping invalid item: {item}") continue print("=" * 70) print(f"Question {index}/{len(questions_data)}") print(f"Task ID: {task_id}") try: submitted_answer = agent(question_text) answers_payload.append({ "task_id": task_id, "submitted_answer": submitted_answer }) results_log.append({ "Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer }) print(f"Submitted answer: {submitted_answer}") except Exception as e: print(f"Agent error on task {task_id}: {e}") results_log.append({ "Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}" }) # --- Make Sure We Have Answers --- if not answers_payload: return ( "Agent did not produce any answers.", pd.DataFrame(results_log) ) # --- Prepare Submission --- submission_data = { "username": username.strip(), "agent_code": agent_code, "answers": answers_payload } print( f"Submitting {len(answers_payload)} answers " f"for user {username}..." ) # --- Submit Answers --- try: response = requests.post( submit_url, json=submission_data, timeout=120 ) response.raise_for_status() result_data = response.json() final_status = ( f"Submission Successful!\n" f"User: {result_data.get('username')}\n" f"Overall Score: " f"{result_data.get('score', 'N/A')}% " f"({result_data.get('correct_count', '?')}/" f"{result_data.get('total_attempted', '?')} correct)\n" f"Message: " f"{result_data.get('message', 'No message received.')}" ) print(final_status) return ( final_status, pd.DataFrame(results_log) ) except requests.exceptions.HTTPError as e: error_detail = ( f"Server responded with status " f"{e.response.status_code}." ) try: error_json = e.response.json() error_detail += ( f" Detail: " f"{error_json.get('detail', e.response.text)}" ) except Exception: error_detail += ( f" Response: {e.response.text[:500]}" ) return ( f"Submission Failed: {error_detail}", pd.DataFrame(results_log) ) except requests.exceptions.Timeout: return ( "Submission Failed: The request timed out.", pd.DataFrame(results_log) ) except requests.exceptions.RequestException as e: return ( f"Submission Failed: Network error - {e}", pd.DataFrame(results_log) ) except Exception as e: return ( f"Unexpected submission error: {e}", pd.DataFrame(results_log) ) # --- Gradio Interface --- with gr.Blocks() as demo: gr.Markdown("# Basic Agent Evaluation Runner") gr.Markdown( """ **Instructions:** 1. Log in to Hugging Face. 2. Click **Run Evaluation & Submit All Answers**. 3. The agent will fetch the benchmark questions. 4. The agent will use web search to research questions. 5. Answers will be submitted automatically. **Note:** Evaluation can take several minutes. """ ) gr.LoginButton() run_button = gr.Button( "Run Evaluation & Submit All Answers" ) status_output = gr.Textbox( label="Run Status / Submission Result", lines=5, interactive=False ) results_table = gr.DataFrame( label="Questions and Agent Answers", wrap=True ) run_button.click( fn=run_and_submit_all, outputs=[ status_output, results_table ] ) # --- Start Application --- if __name__ == "__main__": print("\n" + "-" * 30 + " App Starting " + "-" * 30) space_host = os.getenv("SPACE_HOST") space_id = os.getenv("SPACE_ID") if space_host: print(f"SPACE_HOST: {space_host}") if space_id: print(f"SPACE_ID: {space_id}") print( f"Repository: " f"https://huggingface.co/spaces/{space_id}" ) print("-" * 70) demo.launch( debug=True, share=False )