Update agent.py
Browse files
agent.py
CHANGED
|
@@ -19,13 +19,13 @@ from smolagents import (
|
|
| 19 |
)
|
| 20 |
|
| 21 |
# --------------------------------------------------------------------------- #
|
| 22 |
-
#
|
| 23 |
# --------------------------------------------------------------------------- #
|
| 24 |
|
| 25 |
|
| 26 |
@tool
|
| 27 |
def google_search_fallback(query: str) -> str:
|
| 28 |
-
"""Search
|
| 29 |
|
| 30 |
Args:
|
| 31 |
query: The search query or factual question to look up on Google.
|
|
@@ -52,15 +52,15 @@ def google_search_fallback(query: str) -> str:
|
|
| 52 |
|
| 53 |
@tool
|
| 54 |
def wikipedia_page(title: str) -> str:
|
| 55 |
-
"""Fetch the full text AND all tables of an English Wikipedia page.
|
| 56 |
-
|
| 57 |
|
| 58 |
Args:
|
| 59 |
-
title: The title
|
| 60 |
"""
|
| 61 |
headers = {"User-Agent": "gaia-agent/2.0 (https://huggingface.co)"}
|
| 62 |
try:
|
| 63 |
-
# 1.
|
| 64 |
search_r = requests.get(
|
| 65 |
"https://en.wikipedia.org/w/api.php",
|
| 66 |
params={
|
|
@@ -75,7 +75,7 @@ def wikipedia_page(title: str) -> str:
|
|
| 75 |
search_hits = search_r.get("query", {}).get("search", [])
|
| 76 |
canonical_title = search_hits[0]["title"] if search_hits else title
|
| 77 |
|
| 78 |
-
# 2. Fetch
|
| 79 |
parse_r = requests.get(
|
| 80 |
"https://en.wikipedia.org/w/api.php",
|
| 81 |
params={
|
|
@@ -95,7 +95,6 @@ def wikipedia_page(title: str) -> str:
|
|
| 95 |
html_content = parse_r["parse"]["text"]["*"]
|
| 96 |
soup = BeautifulSoup(html_content, "html.parser")
|
| 97 |
|
| 98 |
-
# Remove style/script/reference noise
|
| 99 |
for tag in soup(["script", "style", "sup"]):
|
| 100 |
tag.decompose()
|
| 101 |
|
|
@@ -119,8 +118,7 @@ def wikipedia_page(title: str) -> str:
|
|
| 119 |
if txt:
|
| 120 |
output_parts.append(txt)
|
| 121 |
|
| 122 |
-
|
| 123 |
-
return full_text[:45000]
|
| 124 |
except Exception as e:
|
| 125 |
return f"Wikipedia error: {e}"
|
| 126 |
|
|
@@ -138,13 +136,17 @@ def read_file(path: str) -> str:
|
|
| 138 |
ext = os.path.splitext(path)[1].lower()
|
| 139 |
if ext == ".csv":
|
| 140 |
df = pd.read_csv(path)
|
| 141 |
-
return
|
|
|
|
|
|
|
|
|
|
| 142 |
if ext in (".xlsx", ".xls"):
|
| 143 |
sheets = pd.read_excel(path, sheet_name=None)
|
| 144 |
out = []
|
| 145 |
for name, df in sheets.items():
|
| 146 |
out.append(
|
| 147 |
-
f"--- Sheet: {name} (Shape: {df.shape}) ---\
|
|
|
|
| 148 |
)
|
| 149 |
return "\n\n".join(out)[:35000]
|
| 150 |
if ext == ".pdf":
|
|
@@ -152,7 +154,7 @@ def read_file(path: str) -> str:
|
|
| 152 |
|
| 153 |
reader = PdfReader(path)
|
| 154 |
return "\n".join(
|
| 155 |
-
f"--- Page {i+1} ---\n{p.extract_text() or ''}"
|
| 156 |
for i, p in enumerate(reader.pages)
|
| 157 |
)[:35000]
|
| 158 |
with open(path, "r", encoding="utf-8", errors="ignore") as f:
|
|
@@ -167,7 +169,7 @@ def transcribe_audio(path: str, question: str = "Transcribe this audio accuratel
|
|
| 167 |
|
| 168 |
Args:
|
| 169 |
path: Local path of the audio file.
|
| 170 |
-
question: Optional specific
|
| 171 |
"""
|
| 172 |
try:
|
| 173 |
if os.getenv("GEMINI_API_KEY"):
|
|
@@ -197,12 +199,12 @@ def transcribe_audio(path: str, question: str = "Transcribe this audio accuratel
|
|
| 197 |
|
| 198 |
@tool
|
| 199 |
def describe_image(path: str, question: str) -> str:
|
| 200 |
-
"""
|
| 201 |
-
Useful for chess boards, charts,
|
| 202 |
|
| 203 |
Args:
|
| 204 |
path: Local path of the image file.
|
| 205 |
-
question:
|
| 206 |
"""
|
| 207 |
try:
|
| 208 |
if os.getenv("GEMINI_API_KEY"):
|
|
@@ -214,7 +216,7 @@ def describe_image(path: str, question: str) -> str:
|
|
| 214 |
model="gemini-2.5-flash",
|
| 215 |
contents=[
|
| 216 |
uploaded,
|
| 217 |
-
f"Examine this image with extreme precision. If it is a chessboard, list every piece and its exact coordinate (a1-h8) for both White and Black first, verify
|
| 218 |
],
|
| 219 |
)
|
| 220 |
return resp.text
|
|
@@ -264,7 +266,7 @@ def describe_image(path: str, question: str) -> str:
|
|
| 264 |
|
| 265 |
@tool
|
| 266 |
def youtube_transcript(url: str) -> str:
|
| 267 |
-
"""Get the timestamped
|
| 268 |
|
| 269 |
Args:
|
| 270 |
url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=XXXX.
|
|
@@ -283,17 +285,17 @@ def youtube_transcript(url: str) -> str:
|
|
| 283 |
parts = YouTubeTranscriptApi.get_transcript(vid)
|
| 284 |
return "\n".join(f"[{int(p['start'])}s] {p['text']}" for p in parts)[:35000]
|
| 285 |
except Exception as e:
|
| 286 |
-
return f"Transcript
|
| 287 |
|
| 288 |
|
| 289 |
@tool
|
| 290 |
def watch_youtube_video(url: str, question: str) -> str:
|
| 291 |
"""Watch a YouTube video directly (both visual frames and audio) using Gemini.
|
| 292 |
-
Use this whenever a question asks about visual
|
| 293 |
|
| 294 |
Args:
|
| 295 |
-
url:
|
| 296 |
-
question:
|
| 297 |
"""
|
| 298 |
if not os.getenv("GEMINI_API_KEY"):
|
| 299 |
return "GEMINI_API_KEY is required to watch YouTube video frames."
|
|
@@ -315,32 +317,32 @@ def watch_youtube_video(url: str, question: str) -> str:
|
|
| 315 |
|
| 316 |
|
| 317 |
# --------------------------------------------------------------------------- #
|
| 318 |
-
# Prompt &
|
| 319 |
# --------------------------------------------------------------------------- #
|
| 320 |
|
| 321 |
-
PROMPT = """You are an elite, meticulous research
|
| 322 |
|
| 323 |
CRITICAL STRATEGY RULES:
|
| 324 |
-
1. Reversed
|
| 325 |
2. Attached Files:
|
| 326 |
-
- Spreadsheets (`.xlsx`, `.csv`):
|
| 327 |
- Audio (`.mp3`, `.wav`): Call `transcribe_audio(path, question)`.
|
| 328 |
- Images (`.png`, `.jpg`): Call `describe_image(path, question)`.
|
| 329 |
-
- Code (`.py`):
|
| 330 |
3. YouTube Videos:
|
| 331 |
-
-
|
| 332 |
-
-
|
| 333 |
4. Wikipedia & Web Search:
|
| 334 |
-
- Use `wikipedia_page` for
|
| 335 |
-
- If `web_search`
|
| 336 |
-
5. Botanical vs. Culinary
|
| 337 |
-
- If
|
| 338 |
|
| 339 |
-
OUTPUT
|
| 340 |
- Call `final_answer` with ONLY the bare answer. No explanation, no "FINAL ANSWER:" prefix.
|
| 341 |
-
-
|
| 342 |
-
-
|
| 343 |
-
-
|
| 344 |
|
| 345 |
{file_note}
|
| 346 |
QUESTION:
|
|
@@ -358,7 +360,7 @@ def clean_answer(ans) -> str:
|
|
| 358 |
|
| 359 |
|
| 360 |
# --------------------------------------------------------------------------- #
|
| 361 |
-
# Model
|
| 362 |
# --------------------------------------------------------------------------- #
|
| 363 |
|
| 364 |
|
|
@@ -366,25 +368,41 @@ def build_model():
|
|
| 366 |
provider = os.getenv("MODEL_PROVIDER", "").lower()
|
| 367 |
model_id = os.getenv("MODEL_ID")
|
| 368 |
|
| 369 |
-
#
|
| 370 |
if provider == "gemini" or (not provider and os.getenv("GEMINI_API_KEY")):
|
| 371 |
-
|
|
|
|
|
|
|
|
|
|
| 372 |
return LiteLLMModel(
|
| 373 |
model_id=model_id or "gemini/gemini-2.5-flash",
|
| 374 |
-
api_key=
|
| 375 |
temperature=0.0,
|
| 376 |
)
|
|
|
|
|
|
|
| 377 |
if provider == "openai" or (not provider and os.getenv("OPENAI_API_KEY")):
|
| 378 |
-
|
|
|
|
|
|
|
|
|
|
| 379 |
return LiteLLMModel(
|
| 380 |
model_id=model_id or "gpt-4o",
|
| 381 |
-
api_key=
|
| 382 |
temperature=0.0,
|
| 383 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 384 |
print("Using Hugging Face InferenceClientModel")
|
| 385 |
return InferenceClientModel(
|
| 386 |
model_id=model_id or "Qwen/Qwen2.5-Coder-32B-Instruct",
|
| 387 |
-
token=
|
| 388 |
temperature=0.0,
|
| 389 |
)
|
| 390 |
|
|
@@ -410,24 +428,20 @@ class GaiaAgent:
|
|
| 410 |
"collections", "itertools", "statistics", "csv", "unicodedata",
|
| 411 |
"requests", "bs4", "openpyxl", "fractions", "string", "pypdf",
|
| 412 |
],
|
| 413 |
-
max_steps=
|
| 414 |
planning_interval=3,
|
| 415 |
verbosity_level=1,
|
| 416 |
)
|
| 417 |
|
| 418 |
def verify_exact_match(self, question: str, raw_answer: str) -> str:
|
| 419 |
-
"""Uses a fast LLM verification pass to strip explanations and enforce GAIA formatting."""
|
| 420 |
cleaned = clean_answer(raw_answer)
|
| 421 |
if not cleaned:
|
| 422 |
return ""
|
| 423 |
-
verify_prompt = f"""You are a strict GAIA benchmark answer
|
| 424 |
-
Given the QUESTION and the AGENT
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
-
|
| 428 |
-
- If the question asks for a count/number, output ONLY digits (e.g. 3).
|
| 429 |
-
- If the question asks for an alphabetized comma-separated list, verify the items are in strict alphabetical order separated by ", ".
|
| 430 |
-
- If the question is written backwards, make sure the answer is to the decoded question (e.g., opposite of "left" is "right").
|
| 431 |
|
| 432 |
QUESTION:
|
| 433 |
{question}
|
|
@@ -435,7 +449,7 @@ QUESTION:
|
|
| 435 |
CANDIDATE ANSWER:
|
| 436 |
{cleaned}
|
| 437 |
|
| 438 |
-
EXACT
|
| 439 |
try:
|
| 440 |
messages = [{"role": "user", "content": [{"type": "text", "text": verify_prompt}]}]
|
| 441 |
resp = self.model(messages)
|
|
|
|
| 19 |
)
|
| 20 |
|
| 21 |
# --------------------------------------------------------------------------- #
|
| 22 |
+
# Tools
|
| 23 |
# --------------------------------------------------------------------------- #
|
| 24 |
|
| 25 |
|
| 26 |
@tool
|
| 27 |
def google_search_fallback(query: str) -> str:
|
| 28 |
+
"""Search Google directly via Gemini Grounding when DuckDuckGo is rate-limited or misses exact facts.
|
| 29 |
|
| 30 |
Args:
|
| 31 |
query: The search query or factual question to look up on Google.
|
|
|
|
| 52 |
|
| 53 |
@tool
|
| 54 |
def wikipedia_page(title: str) -> str:
|
| 55 |
+
"""Fetch the full text AND all HTML tables of an English Wikipedia page.
|
| 56 |
+
Use this for discographies, studio album counts, biographies, and historical tables.
|
| 57 |
|
| 58 |
Args:
|
| 59 |
+
title: The title or search query for the Wikipedia page, e.g. "Mercedes Sosa".
|
| 60 |
"""
|
| 61 |
headers = {"User-Agent": "gaia-agent/2.0 (https://huggingface.co)"}
|
| 62 |
try:
|
| 63 |
+
# 1. Find the closest matching canonical Wikipedia title first
|
| 64 |
search_r = requests.get(
|
| 65 |
"https://en.wikipedia.org/w/api.php",
|
| 66 |
params={
|
|
|
|
| 75 |
search_hits = search_r.get("query", {}).get("search", [])
|
| 76 |
canonical_title = search_hits[0]["title"] if search_hits else title
|
| 77 |
|
| 78 |
+
# 2. Fetch parsed HTML so tables are preserved instead of stripped
|
| 79 |
parse_r = requests.get(
|
| 80 |
"https://en.wikipedia.org/w/api.php",
|
| 81 |
params={
|
|
|
|
| 95 |
html_content = parse_r["parse"]["text"]["*"]
|
| 96 |
soup = BeautifulSoup(html_content, "html.parser")
|
| 97 |
|
|
|
|
| 98 |
for tag in soup(["script", "style", "sup"]):
|
| 99 |
tag.decompose()
|
| 100 |
|
|
|
|
| 118 |
if txt:
|
| 119 |
output_parts.append(txt)
|
| 120 |
|
| 121 |
+
return "\n".join(output_parts)[:45000]
|
|
|
|
| 122 |
except Exception as e:
|
| 123 |
return f"Wikipedia error: {e}"
|
| 124 |
|
|
|
|
| 136 |
ext = os.path.splitext(path)[1].lower()
|
| 137 |
if ext == ".csv":
|
| 138 |
df = pd.read_csv(path)
|
| 139 |
+
return (
|
| 140 |
+
f"Columns: {list(df.columns)}\nShape: {df.shape}\n"
|
| 141 |
+
f"{df.to_string()}\n\nSummary:\n{df.describe(include='all').to_string()}"
|
| 142 |
+
)[:35000]
|
| 143 |
if ext in (".xlsx", ".xls"):
|
| 144 |
sheets = pd.read_excel(path, sheet_name=None)
|
| 145 |
out = []
|
| 146 |
for name, df in sheets.items():
|
| 147 |
out.append(
|
| 148 |
+
f"--- Sheet: {name} (Shape: {df.shape}) ---\n"
|
| 149 |
+
f"Columns: {list(df.columns)}\n{df.to_string()}"
|
| 150 |
)
|
| 151 |
return "\n\n".join(out)[:35000]
|
| 152 |
if ext == ".pdf":
|
|
|
|
| 154 |
|
| 155 |
reader = PdfReader(path)
|
| 156 |
return "\n".join(
|
| 157 |
+
f"--- Page {i + 1} ---\n{p.extract_text() or ''}"
|
| 158 |
for i, p in enumerate(reader.pages)
|
| 159 |
)[:35000]
|
| 160 |
with open(path, "r", encoding="utf-8", errors="ignore") as f:
|
|
|
|
| 169 |
|
| 170 |
Args:
|
| 171 |
path: Local path of the audio file.
|
| 172 |
+
question: Optional specific question or instruction about the audio.
|
| 173 |
"""
|
| 174 |
try:
|
| 175 |
if os.getenv("GEMINI_API_KEY"):
|
|
|
|
| 199 |
|
| 200 |
@tool
|
| 201 |
def describe_image(path: str, question: str) -> str:
|
| 202 |
+
"""Ask a question about an image (png, jpg, jpeg, webp) using a vision model.
|
| 203 |
+
Useful for chess boards, charts, screenshots, and visual puzzles.
|
| 204 |
|
| 205 |
Args:
|
| 206 |
path: Local path of the image file.
|
| 207 |
+
question: What you want to know about the image.
|
| 208 |
"""
|
| 209 |
try:
|
| 210 |
if os.getenv("GEMINI_API_KEY"):
|
|
|
|
| 216 |
model="gemini-2.5-flash",
|
| 217 |
contents=[
|
| 218 |
uploaded,
|
| 219 |
+
f"Examine this image with extreme precision. If it is a chessboard, list every piece and its exact coordinate (a1-h8) for both White and Black first, verify board orientation, and determine the winning move in algebraic notation. Question: {question}",
|
| 220 |
],
|
| 221 |
)
|
| 222 |
return resp.text
|
|
|
|
| 266 |
|
| 267 |
@tool
|
| 268 |
def youtube_transcript(url: str) -> str:
|
| 269 |
+
"""Get the timestamped transcript (captions) of a YouTube video.
|
| 270 |
|
| 271 |
Args:
|
| 272 |
url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=XXXX.
|
|
|
|
| 285 |
parts = YouTubeTranscriptApi.get_transcript(vid)
|
| 286 |
return "\n".join(f"[{int(p['start'])}s] {p['text']}" for p in parts)[:35000]
|
| 287 |
except Exception as e:
|
| 288 |
+
return f"Transcript error: {e} (captions may be unavailable; call `watch_youtube_video` instead!)"
|
| 289 |
|
| 290 |
|
| 291 |
@tool
|
| 292 |
def watch_youtube_video(url: str, question: str) -> str:
|
| 293 |
"""Watch a YouTube video directly (both visual frames and audio) using Gemini.
|
| 294 |
+
Use this whenever a question asks about visual content in a YouTube video (e.g. counting bird species on camera) or when captions are unavailable.
|
| 295 |
|
| 296 |
Args:
|
| 297 |
+
url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=L1vXCYZAYYM.
|
| 298 |
+
question: Specific question about what happens visually or auditorily in the video.
|
| 299 |
"""
|
| 300 |
if not os.getenv("GEMINI_API_KEY"):
|
| 301 |
return "GEMINI_API_KEY is required to watch YouTube video frames."
|
|
|
|
| 317 |
|
| 318 |
|
| 319 |
# --------------------------------------------------------------------------- #
|
| 320 |
+
# Prompt & Exact-Match Post-Processor
|
| 321 |
# --------------------------------------------------------------------------- #
|
| 322 |
|
| 323 |
+
PROMPT = """You are an elite, meticulous research assistant solving a GAIA benchmark question.
|
| 324 |
|
| 325 |
CRITICAL STRATEGY RULES:
|
| 326 |
+
1. Reversed or Encoded Text: If the question text looks reversed (e.g. starts with `.rewsna eht sa`), decode it in Python first, then answer it.
|
| 327 |
2. Attached Files:
|
| 328 |
+
- Spreadsheets (`.xlsx`, `.csv`): Call `read_file(path)` AND write Python `pandas` code to compute exact sums, counts, or filters. Never do mental math!
|
| 329 |
- Audio (`.mp3`, `.wav`): Call `transcribe_audio(path, question)`.
|
| 330 |
- Images (`.png`, `.jpg`): Call `describe_image(path, question)`.
|
| 331 |
+
- Code/Documents (`.py`, `.pdf`, `.txt`): Call `read_file(path)`.
|
| 332 |
3. YouTube Videos:
|
| 333 |
+
- For spoken dialogue, call `youtube_transcript(url)`.
|
| 334 |
+
- For visual questions (e.g., highest number of bird species on camera simultaneously) or when transcripts fail, call `watch_youtube_video(url, question)`.
|
| 335 |
4. Wikipedia & Web Search:
|
| 336 |
+
- Use `wikipedia_page` for studio album counts, discographies, rosters, and biographies.
|
| 337 |
+
- If `web_search` is rate-limited or incomplete, call `google_search_fallback(query)`.
|
| 338 |
+
5. Botanical vs. Culinary Categorization:
|
| 339 |
+
- If asked for strict botanical vegetables (excluding botanical fruits), remember that botanical fruits develop from the flower ovary and contain seeds (bell pepper, zucchini, green beans, corn, plums, peanuts, acorns are botanical fruits/seeds, NOT botanical vegetables). True botanical vegetables are strictly vegetative parts: broccoli, celery, fresh basil, lettuce, sweet potatoes.
|
| 340 |
|
| 341 |
+
OUTPUT RULES (the answer is graded by strict EXACT MATCH):
|
| 342 |
- Call `final_answer` with ONLY the bare answer. No explanation, no "FINAL ANSWER:" prefix.
|
| 343 |
+
- A number: digits only, no commas, no units unless the question explicitly asks for them (e.g. `$89706.00` if USD with two decimal places is requested).
|
| 344 |
+
- A string: no articles (a, an, the) unless part of a proper noun, no abbreviations, no surrounding quotes.
|
| 345 |
+
- A list: comma-separated values with a single space after each comma (`, `), strictly following any alphabetical or numerical order requested.
|
| 346 |
|
| 347 |
{file_note}
|
| 348 |
QUESTION:
|
|
|
|
| 360 |
|
| 361 |
|
| 362 |
# --------------------------------------------------------------------------- #
|
| 363 |
+
# Model & Agent (Auto-Detects API Keys Automatically)
|
| 364 |
# --------------------------------------------------------------------------- #
|
| 365 |
|
| 366 |
|
|
|
|
| 368 |
provider = os.getenv("MODEL_PROVIDER", "").lower()
|
| 369 |
model_id = os.getenv("MODEL_ID")
|
| 370 |
|
| 371 |
+
# Automatically use Gemini if GEMINI_API_KEY is present in Space Secrets
|
| 372 |
if provider == "gemini" or (not provider and os.getenv("GEMINI_API_KEY")):
|
| 373 |
+
key = os.getenv("GEMINI_API_KEY")
|
| 374 |
+
if not key:
|
| 375 |
+
raise RuntimeError("MODEL_PROVIDER=gemini but GEMINI_API_KEY secret is missing.")
|
| 376 |
+
print("Using Gemini 2.5 Flash via LiteLLMModel")
|
| 377 |
return LiteLLMModel(
|
| 378 |
model_id=model_id or "gemini/gemini-2.5-flash",
|
| 379 |
+
api_key=key,
|
| 380 |
temperature=0.0,
|
| 381 |
)
|
| 382 |
+
|
| 383 |
+
# Automatically use OpenAI if OPENAI_API_KEY is present
|
| 384 |
if provider == "openai" or (not provider and os.getenv("OPENAI_API_KEY")):
|
| 385 |
+
key = os.getenv("OPENAI_API_KEY")
|
| 386 |
+
if not key:
|
| 387 |
+
raise RuntimeError("MODEL_PROVIDER=openai but OPENAI_API_KEY secret is missing.")
|
| 388 |
+
print("Using OpenAI via LiteLLMModel")
|
| 389 |
return LiteLLMModel(
|
| 390 |
model_id=model_id or "gpt-4o",
|
| 391 |
+
api_key=key,
|
| 392 |
temperature=0.0,
|
| 393 |
)
|
| 394 |
+
|
| 395 |
+
# Fallback to Hugging Face InferenceClientModel
|
| 396 |
+
token = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
|
| 397 |
+
if not token:
|
| 398 |
+
raise RuntimeError(
|
| 399 |
+
"No GEMINI_API_KEY, OPENAI_API_KEY, or HF_TOKEN found in Space Secrets. "
|
| 400 |
+
"Add GEMINI_API_KEY or HF_TOKEN in Space Settings -> Variables and secrets."
|
| 401 |
+
)
|
| 402 |
print("Using Hugging Face InferenceClientModel")
|
| 403 |
return InferenceClientModel(
|
| 404 |
model_id=model_id or "Qwen/Qwen2.5-Coder-32B-Instruct",
|
| 405 |
+
token=token,
|
| 406 |
temperature=0.0,
|
| 407 |
)
|
| 408 |
|
|
|
|
| 428 |
"collections", "itertools", "statistics", "csv", "unicodedata",
|
| 429 |
"requests", "bs4", "openpyxl", "fractions", "string", "pypdf",
|
| 430 |
],
|
| 431 |
+
max_steps=14,
|
| 432 |
planning_interval=3,
|
| 433 |
verbosity_level=1,
|
| 434 |
)
|
| 435 |
|
| 436 |
def verify_exact_match(self, question: str, raw_answer: str) -> str:
|
|
|
|
| 437 |
cleaned = clean_answer(raw_answer)
|
| 438 |
if not cleaned:
|
| 439 |
return ""
|
| 440 |
+
verify_prompt = f"""You are a strict GAIA benchmark answer extractor.
|
| 441 |
+
Given the QUESTION and the AGENT CANDIDATE ANSWER, output ONLY the exact final string required for strict exact-match grading.
|
| 442 |
+
- No explanation, no prefix, no quotes.
|
| 443 |
+
- Digits only for counts/numbers unless units/currency are explicitly asked.
|
| 444 |
+
- Verify strict alphabetical order and ', ' spacing if a list is requested.
|
|
|
|
|
|
|
|
|
|
| 445 |
|
| 446 |
QUESTION:
|
| 447 |
{question}
|
|
|
|
| 449 |
CANDIDATE ANSWER:
|
| 450 |
{cleaned}
|
| 451 |
|
| 452 |
+
EXACT FINAL ANSWER:"""
|
| 453 |
try:
|
| 454 |
messages = [{"role": "user", "content": [{"type": "text", "text": verify_prompt}]}]
|
| 455 |
resp = self.model(messages)
|