File size: 3,423 Bytes
d87cd44 90bfaa0 0967681 90bfaa0 d87cd44 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 0967681 d87cd44 0967681 d87cd44 0967681 90bfaa0 0967681 d87cd44 90bfaa0 d87cd44 90bfaa0 d87cd44 90bfaa0 0967681 90bfaa0 0967681 90bfaa0 d87cd44 0967681 d87cd44 90bfaa0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import os
import gradio as gr
import pandas as pd
import requests
from transformers import pipeline
# --- Local LLM Agent ---
class LocalLLMAgent:
def __init__(self):
print("Loading local model...")
self.generator = pipeline(
"text-generation",
model="distilgpt2", # small, fast
device=-1 # CPU only
)
def __call__(self, question: str) -> str:
try:
outputs = self.generator(
question,
max_length=50,
do_sample=False,
num_return_sequences=1,
)
return outputs[0]["generated_text"].strip()
except Exception as e:
print(f"[LocalLLMAgent Error] {e}")
return "Error: Could not generate answer."
# --- Dummy Fallback Agent ---
class DummyAgent:
def __call__(self, question: str) -> str:
return "Error: Could not generate answer."
# --- GAIA Submission Logic ---
def run_and_submit_all(profile: gr.OAuthProfile | None):
GAIA_API = "https://agents-course-unit4-scoring.hf.space"
space_id = os.getenv("SPACE_ID", "XeroDN/final_assignment")
username = profile.username if profile else "anonymous"
try:
agent = LocalLLMAgent()
except Exception as e:
print("Falling back to DummyAgent:", e)
agent = DummyAgent()
# Fetch GAIA questions
try:
qres = requests.get(f"{GAIA_API}/questions", timeout=15)
qres.raise_for_status()
questions = qres.json()
except Exception as e:
return f"❌ Failed to fetch questions: {e}", pd.DataFrame()
# Generate answers
results_log = []
answers_payload = []
for item in questions:
q = item.get("question")
tid = item.get("task_id")
if not q or not tid:
continue
answer = agent(q)
results_log.append({"Task ID": tid, "Question": q, "Submitted Answer": answer})
answers_payload.append({"task_id": tid, "submitted_answer": answer})
if not answers_payload:
return "⚠️ No answers generated.", pd.DataFrame()
# Submit answers
try:
submission = {
"username": username,
"agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
"answers": answers_payload
}
sres = requests.post(f"{GAIA_API}/submit", json=submission, timeout=60)
sres.raise_for_status()
result = sres.json()
score = result.get("score", "?")
correct = result.get("correct_count", "?")
total = result.get("total_attempted", "?")
summary = f"✅ Submission Successful!\nUser: {username}\nScore: {score}% ({correct}/{total})"
return summary, pd.DataFrame(results_log)
except Exception as e:
return f"❌ Submission failed: {e}", pd.DataFrame(results_log)
# --- Gradio UI ---
with gr.Blocks() as demo:
gr.Markdown("## 🤖 GAIA Benchmark Agent Submission (Local LLM, no API key)")
gr.Markdown("Click below to log in and run your agent on the GAIA benchmark.")
login_btn = gr.LoginButton()
run_btn = gr.Button("🚀 Run Evaluation & Submit All Answers")
status = gr.Textbox(label="Status", lines=6)
table = gr.DataFrame(label="Results")
run_btn.click(fn=run_and_submit_all, inputs=[login_btn], outputs=[status, table])
if __name__ == "__main__":
demo.launch()
|