DuyKhai commited on
Commit
d35256e
·
verified ·
1 Parent(s): f82bcce

Update agent.py

Browse files
Files changed (1) hide show
  1. agent.py +70 -56
agent.py CHANGED
@@ -19,13 +19,13 @@ from smolagents import (
19
  )
20
 
21
  # --------------------------------------------------------------------------- #
22
- # Upgraded Tools
23
  # --------------------------------------------------------------------------- #
24
 
25
 
26
  @tool
27
  def google_search_fallback(query: str) -> str:
28
- """Search the web using Google Search grounding via Gemini when DuckDuckGo is rate-limited or misses exact facts.
29
 
30
  Args:
31
  query: The search query or factual question to look up on Google.
@@ -52,15 +52,15 @@ def google_search_fallback(query: str) -> str:
52
 
53
  @tool
54
  def wikipedia_page(title: str) -> str:
55
- """Fetch the full text AND all tables of an English Wikipedia page.
56
- Crucial for discographies, studio album counts, biographies, and historical tables.
57
 
58
  Args:
59
- title: The title of the Wikipedia page, e.g. "Mercedes Sosa".
60
  """
61
  headers = {"User-Agent": "gaia-agent/2.0 (https://huggingface.co)"}
62
  try:
63
- # 1. Search for the closest matching canonical page title first
64
  search_r = requests.get(
65
  "https://en.wikipedia.org/w/api.php",
66
  params={
@@ -75,7 +75,7 @@ def wikipedia_page(title: str) -> str:
75
  search_hits = search_r.get("query", {}).get("search", [])
76
  canonical_title = search_hits[0]["title"] if search_hits else title
77
 
78
- # 2. Fetch full parsed HTML so tables (discographies, lists) are NOT stripped
79
  parse_r = requests.get(
80
  "https://en.wikipedia.org/w/api.php",
81
  params={
@@ -95,7 +95,6 @@ def wikipedia_page(title: str) -> str:
95
  html_content = parse_r["parse"]["text"]["*"]
96
  soup = BeautifulSoup(html_content, "html.parser")
97
 
98
- # Remove style/script/reference noise
99
  for tag in soup(["script", "style", "sup"]):
100
  tag.decompose()
101
 
@@ -119,8 +118,7 @@ def wikipedia_page(title: str) -> str:
119
  if txt:
120
  output_parts.append(txt)
121
 
122
- full_text = "\n".join(output_parts)
123
- return full_text[:45000]
124
  except Exception as e:
125
  return f"Wikipedia error: {e}"
126
 
@@ -138,13 +136,17 @@ def read_file(path: str) -> str:
138
  ext = os.path.splitext(path)[1].lower()
139
  if ext == ".csv":
140
  df = pd.read_csv(path)
141
- return f"Columns: {list(df.columns)}\nShape: {df.shape}\n{df.to_string()}\n\nSummary:\n{df.describe(include='all').to_string()}"[:35000]
 
 
 
142
  if ext in (".xlsx", ".xls"):
143
  sheets = pd.read_excel(path, sheet_name=None)
144
  out = []
145
  for name, df in sheets.items():
146
  out.append(
147
- f"--- Sheet: {name} (Shape: {df.shape}) ---\nColumns: {list(df.columns)}\n{df.to_string()}"
 
148
  )
149
  return "\n\n".join(out)[:35000]
150
  if ext == ".pdf":
@@ -152,7 +154,7 @@ def read_file(path: str) -> str:
152
 
153
  reader = PdfReader(path)
154
  return "\n".join(
155
- f"--- Page {i+1} ---\n{p.extract_text() or ''}"
156
  for i, p in enumerate(reader.pages)
157
  )[:35000]
158
  with open(path, "r", encoding="utf-8", errors="ignore") as f:
@@ -167,7 +169,7 @@ def transcribe_audio(path: str, question: str = "Transcribe this audio accuratel
167
 
168
  Args:
169
  path: Local path of the audio file.
170
- question: Optional specific instruction (default is full word-for-word transcription).
171
  """
172
  try:
173
  if os.getenv("GEMINI_API_KEY"):
@@ -197,12 +199,12 @@ def transcribe_audio(path: str, question: str = "Transcribe this audio accuratel
197
 
198
  @tool
199
  def describe_image(path: str, question: str) -> str:
200
- """Analyze an image (png, jpg, jpeg, webp) carefully using a vision model.
201
- Useful for chess boards, charts, diagrams, screenshots, and visual puzzles.
202
 
203
  Args:
204
  path: Local path of the image file.
205
- question: Specific question to answer about the image.
206
  """
207
  try:
208
  if os.getenv("GEMINI_API_KEY"):
@@ -214,7 +216,7 @@ def describe_image(path: str, question: str) -> str:
214
  model="gemini-2.5-flash",
215
  contents=[
216
  uploaded,
217
- f"Examine this image with extreme precision. If it is a chessboard, list every piece and its exact coordinate (a1-h8) for both White and Black first, verify the board orientation, and calculate the winning move in standard algebraic notation. Question: {question}",
218
  ],
219
  )
220
  return resp.text
@@ -264,7 +266,7 @@ def describe_image(path: str, question: str) -> str:
264
 
265
  @tool
266
  def youtube_transcript(url: str) -> str:
267
- """Get the timestamped text transcript (captions) of a YouTube video.
268
 
269
  Args:
270
  url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=XXXX.
@@ -283,17 +285,17 @@ def youtube_transcript(url: str) -> str:
283
  parts = YouTubeTranscriptApi.get_transcript(vid)
284
  return "\n".join(f"[{int(p['start'])}s] {p['text']}" for p in parts)[:35000]
285
  except Exception as e:
286
- return f"Transcript unavailable: {e}. Use `watch_youtube_video` tool instead to inspect the video directly!"
287
 
288
 
289
  @tool
290
  def watch_youtube_video(url: str, question: str) -> str:
291
  """Watch a YouTube video directly (both visual frames and audio) using Gemini.
292
- Use this whenever a question asks about visual events in a YouTube video (e.g. counting bird species on camera) or when captions are unavailable.
293
 
294
  Args:
295
- url: The full YouTube URL (e.g. https://www.youtube.com/watch?v=L1vXCYZAYYM).
296
- question: What to count, transcribe, or observe in the video.
297
  """
298
  if not os.getenv("GEMINI_API_KEY"):
299
  return "GEMINI_API_KEY is required to watch YouTube video frames."
@@ -315,32 +317,32 @@ def watch_youtube_video(url: str, question: str) -> str:
315
 
316
 
317
  # --------------------------------------------------------------------------- #
318
- # Prompt & Answer Post-Processor
319
  # --------------------------------------------------------------------------- #
320
 
321
- PROMPT = """You are an elite, meticulous research agent solving a GAIA benchmark question.
322
 
323
  CRITICAL STRATEGY RULES:
324
- 1. Reversed / Encoded Text: If the question text is reversed (e.g. starts with `.rewsna eht sa`), decode it first using Python before answering.
325
  2. Attached Files:
326
- - Spreadsheets (`.xlsx`, `.csv`): Use `read_file` AND write Python `pandas` code to compute exact sums/averages/filters. Never do mental math!
327
  - Audio (`.mp3`, `.wav`): Call `transcribe_audio(path, question)`.
328
  - Images (`.png`, `.jpg`): Call `describe_image(path, question)`.
329
- - Code (`.py`): Read the file with `read_file(path)` and execute or simulate the code in Python to get the exact output.
330
  3. YouTube Videos:
331
- - If the question asks about spoken dialogue, call `youtube_transcript(url)` (and fall back to `watch_youtube_video(url, question)` if needed).
332
- - If the question asks about VISUAL objects on camera (e.g. counting bird species simultaneously on camera), MUST call `watch_youtube_video(url, question)`.
333
  4. Wikipedia & Web Search:
334
- - Use `wikipedia_page` for Wikipedia articles, studio album counts, and tables.
335
- - If `web_search` fails or gives incomplete snippets, call `google_search_fallback(query)`.
336
- 5. Botanical vs. Culinary Questions:
337
- - If a question asks for strict botanical vegetables (excluding botanical fruits), remember that botanical fruits develop from the flower's ovary and contain seeds (e.g. bell pepper, zucchini, green beans, corn, plums, peanuts, acorns are ALL botanical fruits/seeds, NOT botanical vegetables!). True botanical vegetables are strictly vegetative parts: roots/tubers (sweet potatoes), stems/petioles (celery), leaves (fresh basil, lettuce), and flower buds (broccoli).
338
 
339
- OUTPUT FORMAT RULES (GRADED BY STRICT EXACT MATCH):
340
  - Call `final_answer` with ONLY the bare answer. No explanation, no "FINAL ANSWER:" prefix.
341
- - Number: digits only (e.g. `4`, not `4 albums` or `four`), no commas unless requested. If asked for USD with two decimal places, format as `$89706.00` (or plain digits if no symbol requested).
342
- - String: exact name/word only, no surrounding quotes, no articles (`a`, `an`, `the`) unless part of a proper title.
343
- - List: comma-separated with a single space after each comma (e.g., `broccoli, celery, fresh basil, lettuce, sweet potatoes`), strictly following any requested alphabetical ordering.
344
 
345
  {file_note}
346
  QUESTION:
@@ -358,7 +360,7 @@ def clean_answer(ans) -> str:
358
 
359
 
360
  # --------------------------------------------------------------------------- #
361
- # Model Builder (Auto-Detects API Keys Automatically!)
362
  # --------------------------------------------------------------------------- #
363
 
364
 
@@ -366,25 +368,41 @@ def build_model():
366
  provider = os.getenv("MODEL_PROVIDER", "").lower()
367
  model_id = os.getenv("MODEL_ID")
368
 
369
- # Auto-detect Gemini first if GEMINI_API_KEY is present in Space Secrets
370
  if provider == "gemini" or (not provider and os.getenv("GEMINI_API_KEY")):
371
- print("Using Gemini model via LiteLLMModel")
 
 
 
372
  return LiteLLMModel(
373
  model_id=model_id or "gemini/gemini-2.5-flash",
374
- api_key=os.getenv("GEMINI_API_KEY"),
375
  temperature=0.0,
376
  )
 
 
377
  if provider == "openai" or (not provider and os.getenv("OPENAI_API_KEY")):
378
- print("Using OpenAI model via LiteLLMModel")
 
 
 
379
  return LiteLLMModel(
380
  model_id=model_id or "gpt-4o",
381
- api_key=os.getenv("OPENAI_API_KEY"),
382
  temperature=0.0,
383
  )
 
 
 
 
 
 
 
 
384
  print("Using Hugging Face InferenceClientModel")
385
  return InferenceClientModel(
386
  model_id=model_id or "Qwen/Qwen2.5-Coder-32B-Instruct",
387
- token=os.getenv("HF_TOKEN"),
388
  temperature=0.0,
389
  )
390
 
@@ -410,24 +428,20 @@ class GaiaAgent:
410
  "collections", "itertools", "statistics", "csv", "unicodedata",
411
  "requests", "bs4", "openpyxl", "fractions", "string", "pypdf",
412
  ],
413
- max_steps=15,
414
  planning_interval=3,
415
  verbosity_level=1,
416
  )
417
 
418
  def verify_exact_match(self, question: str, raw_answer: str) -> str:
419
- """Uses a fast LLM verification pass to strip explanations and enforce GAIA formatting."""
420
  cleaned = clean_answer(raw_answer)
421
  if not cleaned:
422
  return ""
423
- verify_prompt = f"""You are a strict GAIA benchmark answer formatter.
424
- Given the QUESTION and the AGENT'S CANDIDATE ANSWER, output ONLY the exact final string required for exact-match grading.
425
-
426
- Formatting rules:
427
- - Output ONLY the final answer and nothing else (no prefixes, no explanations, no quotes).
428
- - If the question asks for a count/number, output ONLY digits (e.g. 3).
429
- - If the question asks for an alphabetized comma-separated list, verify the items are in strict alphabetical order separated by ", ".
430
- - If the question is written backwards, make sure the answer is to the decoded question (e.g., opposite of "left" is "right").
431
 
432
  QUESTION:
433
  {question}
@@ -435,7 +449,7 @@ QUESTION:
435
  CANDIDATE ANSWER:
436
  {cleaned}
437
 
438
- EXACT MATCH ANSWER:"""
439
  try:
440
  messages = [{"role": "user", "content": [{"type": "text", "text": verify_prompt}]}]
441
  resp = self.model(messages)
 
19
  )
20
 
21
  # --------------------------------------------------------------------------- #
22
+ # Tools
23
  # --------------------------------------------------------------------------- #
24
 
25
 
26
  @tool
27
  def google_search_fallback(query: str) -> str:
28
+ """Search Google directly via Gemini Grounding when DuckDuckGo is rate-limited or misses exact facts.
29
 
30
  Args:
31
  query: The search query or factual question to look up on Google.
 
52
 
53
  @tool
54
  def wikipedia_page(title: str) -> str:
55
+ """Fetch the full text AND all HTML tables of an English Wikipedia page.
56
+ Use this for discographies, studio album counts, biographies, and historical tables.
57
 
58
  Args:
59
+ title: The title or search query for the Wikipedia page, e.g. "Mercedes Sosa".
60
  """
61
  headers = {"User-Agent": "gaia-agent/2.0 (https://huggingface.co)"}
62
  try:
63
+ # 1. Find the closest matching canonical Wikipedia title first
64
  search_r = requests.get(
65
  "https://en.wikipedia.org/w/api.php",
66
  params={
 
75
  search_hits = search_r.get("query", {}).get("search", [])
76
  canonical_title = search_hits[0]["title"] if search_hits else title
77
 
78
+ # 2. Fetch parsed HTML so tables are preserved instead of stripped
79
  parse_r = requests.get(
80
  "https://en.wikipedia.org/w/api.php",
81
  params={
 
95
  html_content = parse_r["parse"]["text"]["*"]
96
  soup = BeautifulSoup(html_content, "html.parser")
97
 
 
98
  for tag in soup(["script", "style", "sup"]):
99
  tag.decompose()
100
 
 
118
  if txt:
119
  output_parts.append(txt)
120
 
121
+ return "\n".join(output_parts)[:45000]
 
122
  except Exception as e:
123
  return f"Wikipedia error: {e}"
124
 
 
136
  ext = os.path.splitext(path)[1].lower()
137
  if ext == ".csv":
138
  df = pd.read_csv(path)
139
+ return (
140
+ f"Columns: {list(df.columns)}\nShape: {df.shape}\n"
141
+ f"{df.to_string()}\n\nSummary:\n{df.describe(include='all').to_string()}"
142
+ )[:35000]
143
  if ext in (".xlsx", ".xls"):
144
  sheets = pd.read_excel(path, sheet_name=None)
145
  out = []
146
  for name, df in sheets.items():
147
  out.append(
148
+ f"--- Sheet: {name} (Shape: {df.shape}) ---\n"
149
+ f"Columns: {list(df.columns)}\n{df.to_string()}"
150
  )
151
  return "\n\n".join(out)[:35000]
152
  if ext == ".pdf":
 
154
 
155
  reader = PdfReader(path)
156
  return "\n".join(
157
+ f"--- Page {i + 1} ---\n{p.extract_text() or ''}"
158
  for i, p in enumerate(reader.pages)
159
  )[:35000]
160
  with open(path, "r", encoding="utf-8", errors="ignore") as f:
 
169
 
170
  Args:
171
  path: Local path of the audio file.
172
+ question: Optional specific question or instruction about the audio.
173
  """
174
  try:
175
  if os.getenv("GEMINI_API_KEY"):
 
199
 
200
  @tool
201
  def describe_image(path: str, question: str) -> str:
202
+ """Ask a question about an image (png, jpg, jpeg, webp) using a vision model.
203
+ Useful for chess boards, charts, screenshots, and visual puzzles.
204
 
205
  Args:
206
  path: Local path of the image file.
207
+ question: What you want to know about the image.
208
  """
209
  try:
210
  if os.getenv("GEMINI_API_KEY"):
 
216
  model="gemini-2.5-flash",
217
  contents=[
218
  uploaded,
219
+ f"Examine this image with extreme precision. If it is a chessboard, list every piece and its exact coordinate (a1-h8) for both White and Black first, verify board orientation, and determine the winning move in algebraic notation. Question: {question}",
220
  ],
221
  )
222
  return resp.text
 
266
 
267
  @tool
268
  def youtube_transcript(url: str) -> str:
269
+ """Get the timestamped transcript (captions) of a YouTube video.
270
 
271
  Args:
272
  url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=XXXX.
 
285
  parts = YouTubeTranscriptApi.get_transcript(vid)
286
  return "\n".join(f"[{int(p['start'])}s] {p['text']}" for p in parts)[:35000]
287
  except Exception as e:
288
+ return f"Transcript error: {e} (captions may be unavailable; call `watch_youtube_video` instead!)"
289
 
290
 
291
  @tool
292
  def watch_youtube_video(url: str, question: str) -> str:
293
  """Watch a YouTube video directly (both visual frames and audio) using Gemini.
294
+ Use this whenever a question asks about visual content in a YouTube video (e.g. counting bird species on camera) or when captions are unavailable.
295
 
296
  Args:
297
+ url: Full YouTube URL, e.g. https://www.youtube.com/watch?v=L1vXCYZAYYM.
298
+ question: Specific question about what happens visually or auditorily in the video.
299
  """
300
  if not os.getenv("GEMINI_API_KEY"):
301
  return "GEMINI_API_KEY is required to watch YouTube video frames."
 
317
 
318
 
319
  # --------------------------------------------------------------------------- #
320
+ # Prompt & Exact-Match Post-Processor
321
  # --------------------------------------------------------------------------- #
322
 
323
+ PROMPT = """You are an elite, meticulous research assistant solving a GAIA benchmark question.
324
 
325
  CRITICAL STRATEGY RULES:
326
+ 1. Reversed or Encoded Text: If the question text looks reversed (e.g. starts with `.rewsna eht sa`), decode it in Python first, then answer it.
327
  2. Attached Files:
328
+ - Spreadsheets (`.xlsx`, `.csv`): Call `read_file(path)` AND write Python `pandas` code to compute exact sums, counts, or filters. Never do mental math!
329
  - Audio (`.mp3`, `.wav`): Call `transcribe_audio(path, question)`.
330
  - Images (`.png`, `.jpg`): Call `describe_image(path, question)`.
331
+ - Code/Documents (`.py`, `.pdf`, `.txt`): Call `read_file(path)`.
332
  3. YouTube Videos:
333
+ - For spoken dialogue, call `youtube_transcript(url)`.
334
+ - For visual questions (e.g., highest number of bird species on camera simultaneously) or when transcripts fail, call `watch_youtube_video(url, question)`.
335
  4. Wikipedia & Web Search:
336
+ - Use `wikipedia_page` for studio album counts, discographies, rosters, and biographies.
337
+ - If `web_search` is rate-limited or incomplete, call `google_search_fallback(query)`.
338
+ 5. Botanical vs. Culinary Categorization:
339
+ - If asked for strict botanical vegetables (excluding botanical fruits), remember that botanical fruits develop from the flower ovary and contain seeds (bell pepper, zucchini, green beans, corn, plums, peanuts, acorns are botanical fruits/seeds, NOT botanical vegetables). True botanical vegetables are strictly vegetative parts: broccoli, celery, fresh basil, lettuce, sweet potatoes.
340
 
341
+ OUTPUT RULES (the answer is graded by strict EXACT MATCH):
342
  - Call `final_answer` with ONLY the bare answer. No explanation, no "FINAL ANSWER:" prefix.
343
+ - A number: digits only, no commas, no units unless the question explicitly asks for them (e.g. `$89706.00` if USD with two decimal places is requested).
344
+ - A string: no articles (a, an, the) unless part of a proper noun, no abbreviations, no surrounding quotes.
345
+ - A list: comma-separated values with a single space after each comma (`, `), strictly following any alphabetical or numerical order requested.
346
 
347
  {file_note}
348
  QUESTION:
 
360
 
361
 
362
  # --------------------------------------------------------------------------- #
363
+ # Model & Agent (Auto-Detects API Keys Automatically)
364
  # --------------------------------------------------------------------------- #
365
 
366
 
 
368
  provider = os.getenv("MODEL_PROVIDER", "").lower()
369
  model_id = os.getenv("MODEL_ID")
370
 
371
+ # Automatically use Gemini if GEMINI_API_KEY is present in Space Secrets
372
  if provider == "gemini" or (not provider and os.getenv("GEMINI_API_KEY")):
373
+ key = os.getenv("GEMINI_API_KEY")
374
+ if not key:
375
+ raise RuntimeError("MODEL_PROVIDER=gemini but GEMINI_API_KEY secret is missing.")
376
+ print("Using Gemini 2.5 Flash via LiteLLMModel")
377
  return LiteLLMModel(
378
  model_id=model_id or "gemini/gemini-2.5-flash",
379
+ api_key=key,
380
  temperature=0.0,
381
  )
382
+
383
+ # Automatically use OpenAI if OPENAI_API_KEY is present
384
  if provider == "openai" or (not provider and os.getenv("OPENAI_API_KEY")):
385
+ key = os.getenv("OPENAI_API_KEY")
386
+ if not key:
387
+ raise RuntimeError("MODEL_PROVIDER=openai but OPENAI_API_KEY secret is missing.")
388
+ print("Using OpenAI via LiteLLMModel")
389
  return LiteLLMModel(
390
  model_id=model_id or "gpt-4o",
391
+ api_key=key,
392
  temperature=0.0,
393
  )
394
+
395
+ # Fallback to Hugging Face InferenceClientModel
396
+ token = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACEHUB_API_TOKEN")
397
+ if not token:
398
+ raise RuntimeError(
399
+ "No GEMINI_API_KEY, OPENAI_API_KEY, or HF_TOKEN found in Space Secrets. "
400
+ "Add GEMINI_API_KEY or HF_TOKEN in Space Settings -> Variables and secrets."
401
+ )
402
  print("Using Hugging Face InferenceClientModel")
403
  return InferenceClientModel(
404
  model_id=model_id or "Qwen/Qwen2.5-Coder-32B-Instruct",
405
+ token=token,
406
  temperature=0.0,
407
  )
408
 
 
428
  "collections", "itertools", "statistics", "csv", "unicodedata",
429
  "requests", "bs4", "openpyxl", "fractions", "string", "pypdf",
430
  ],
431
+ max_steps=14,
432
  planning_interval=3,
433
  verbosity_level=1,
434
  )
435
 
436
  def verify_exact_match(self, question: str, raw_answer: str) -> str:
 
437
  cleaned = clean_answer(raw_answer)
438
  if not cleaned:
439
  return ""
440
+ verify_prompt = f"""You are a strict GAIA benchmark answer extractor.
441
+ Given the QUESTION and the AGENT CANDIDATE ANSWER, output ONLY the exact final string required for strict exact-match grading.
442
+ - No explanation, no prefix, no quotes.
443
+ - Digits only for counts/numbers unless units/currency are explicitly asked.
444
+ - Verify strict alphabetical order and ', ' spacing if a list is requested.
 
 
 
445
 
446
  QUESTION:
447
  {question}
 
449
  CANDIDATE ANSWER:
450
  {cleaned}
451
 
452
+ EXACT FINAL ANSWER:"""
453
  try:
454
  messages = [{"role": "user", "content": [{"type": "text", "text": verify_prompt}]}]
455
  resp = self.model(messages)