| import os |
| import gradio as gr |
| import pandas as pd |
| import requests |
| from transformers import pipeline |
|
|
| |
| class LocalLLMAgent: |
| def __init__(self): |
| print("Loading local model...") |
| self.generator = pipeline( |
| "text-generation", |
| model="distilgpt2", |
| device=-1 |
| ) |
|
|
| def __call__(self, question: str) -> str: |
| try: |
| outputs = self.generator( |
| question, |
| max_length=50, |
| do_sample=False, |
| num_return_sequences=1, |
| ) |
| return outputs[0]["generated_text"].strip() |
| except Exception as e: |
| print(f"[LocalLLMAgent Error] {e}") |
| return "Error: Could not generate answer." |
|
|
|
|
| |
| class DummyAgent: |
| def __call__(self, question: str) -> str: |
| return "Error: Could not generate answer." |
|
|
|
|
| |
| def run_and_submit_all(profile: gr.OAuthProfile | None): |
| GAIA_API = "https://agents-course-unit4-scoring.hf.space" |
| space_id = os.getenv("SPACE_ID", "XeroDN/final_assignment") |
| username = profile.username if profile else "anonymous" |
|
|
| try: |
| agent = LocalLLMAgent() |
| except Exception as e: |
| print("Falling back to DummyAgent:", e) |
| agent = DummyAgent() |
|
|
| |
| try: |
| qres = requests.get(f"{GAIA_API}/questions", timeout=15) |
| qres.raise_for_status() |
| questions = qres.json() |
| except Exception as e: |
| return f"❌ Failed to fetch questions: {e}", pd.DataFrame() |
|
|
| |
| results_log = [] |
| answers_payload = [] |
| for item in questions: |
| q = item.get("question") |
| tid = item.get("task_id") |
| if not q or not tid: |
| continue |
| answer = agent(q) |
| results_log.append({"Task ID": tid, "Question": q, "Submitted Answer": answer}) |
| answers_payload.append({"task_id": tid, "submitted_answer": answer}) |
|
|
| if not answers_payload: |
| return "⚠️ No answers generated.", pd.DataFrame() |
|
|
| |
| try: |
| submission = { |
| "username": username, |
| "agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main", |
| "answers": answers_payload |
| } |
| sres = requests.post(f"{GAIA_API}/submit", json=submission, timeout=60) |
| sres.raise_for_status() |
| result = sres.json() |
| score = result.get("score", "?") |
| correct = result.get("correct_count", "?") |
| total = result.get("total_attempted", "?") |
| summary = f"✅ Submission Successful!\nUser: {username}\nScore: {score}% ({correct}/{total})" |
| return summary, pd.DataFrame(results_log) |
| except Exception as e: |
| return f"❌ Submission failed: {e}", pd.DataFrame(results_log) |
|
|
| |
| with gr.Blocks() as demo: |
| gr.Markdown("## 🤖 GAIA Benchmark Agent Submission (Local LLM, no API key)") |
| gr.Markdown("Click below to log in and run your agent on the GAIA benchmark.") |
|
|
| login_btn = gr.LoginButton() |
| run_btn = gr.Button("🚀 Run Evaluation & Submit All Answers") |
| status = gr.Textbox(label="Status", lines=6) |
| table = gr.DataFrame(label="Results") |
|
|
| run_btn.click(fn=run_and_submit_all, inputs=[login_btn], outputs=[status, table]) |
|
|
| if __name__ == "__main__": |
| demo.launch() |
|
|
