Prashhaannt commited on
Commit
0c19db9
·
verified ·
1 Parent(s): 02cc6ca

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +497 -333
app.py CHANGED
@@ -1,5 +1,6 @@
1
  import os
2
  import re
 
3
  import tempfile
4
  from pathlib import Path
5
  from urllib.parse import unquote, urlparse, parse_qs
@@ -9,37 +10,44 @@ import pandas as pd
9
  import requests
10
  from pypdf import PdfReader
11
  from youtube_transcript_api import YouTubeTranscriptApi
 
12
  from smolagents import (
13
  CodeAgent,
14
  DuckDuckGoSearchTool,
15
  InferenceClientModel,
16
  VisitWebpageTool,
 
17
  )
18
 
19
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
20
- MODEL_ID = "Qwen/Qwen2.5-Coder-32B-Instruct"
 
21
 
22
 
23
- # ---------------------------------------------------------------------
24
- # Helpers
25
- # ---------------------------------------------------------------------
26
-
27
  def clean_answer(value: str) -> str:
28
- value = str(value).strip()
29
 
30
- for prefix in ("final answer:", "final answer", "answer:"):
 
 
 
 
 
31
  if value.lower().startswith(prefix):
32
  value = value[len(prefix):].lstrip(": \n\t").strip()
33
  break
34
 
35
- if (
36
- len(value) >= 2
37
- and value[0] == value[-1]
38
- and value[0] in {'"', "'"}
39
- ):
40
  value = value[1:-1].strip()
41
 
42
- # Block obvious agent-trace leakage from exact-match submission.
 
 
 
 
43
  bad_markers = (
44
  "Thought:",
45
  "Calling tools:",
@@ -47,153 +55,57 @@ def clean_answer(value: str) -> str:
47
  "</code>",
48
  "python_interpreter",
49
  )
50
- if any(marker in value for marker in bad_markers):
51
  return ""
52
 
53
  return value
54
 
55
 
56
- def _filename_from_response(response: requests.Response, task_id: str) -> str:
57
- header = response.headers.get("content-disposition", "")
58
-
59
- match = re.search(r"filename\*=UTF-8''([^;]+)", header, flags=re.IGNORECASE)
60
- if match:
61
- return Path(unquote(match.group(1))).name
62
-
63
- match = re.search(r'filename="?([^";]+)"?', header, flags=re.IGNORECASE)
64
- if match:
65
- return Path(match.group(1).strip()).name
66
-
67
- content_type = response.headers.get("content-type", "").split(";")[0].lower()
68
-
69
- ext_map = {
70
- "application/pdf": ".pdf",
71
- "text/csv": ".csv",
72
- "application/json": ".json",
73
- "text/plain": ".txt",
74
- "audio/mpeg": ".mp3",
75
- "audio/wav": ".wav",
76
- "image/png": ".png",
77
- "image/jpeg": ".jpg",
78
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
79
- "application/vnd.ms-excel": ".xls",
80
- }
81
-
82
- return f"{task_id}{ext_map.get(content_type, '.bin')}"
83
-
84
-
85
- def download_task_file(api_url: str, task_id: str) -> str | None:
86
- url = f"{api_url}/files/{task_id}"
87
-
88
- try:
89
- response = requests.get(url, timeout=30)
90
- if response.status_code == 404:
91
- return None
92
-
93
- response.raise_for_status()
94
-
95
- if not response.content:
96
- return None
97
-
98
- filename = _filename_from_response(response, task_id)
99
- temp_dir = Path(tempfile.mkdtemp(prefix=f"gaia_{task_id[:8]}_"))
100
- path = temp_dir / filename
101
- path.write_bytes(response.content)
102
- return str(path)
103
-
104
- except Exception as exc:
105
- print(f"Attachment unavailable for {task_id}: {exc}")
106
- return None
107
-
108
-
109
- def extract_attachment_content(file_path: str | None) -> str:
110
- if not file_path:
111
- return ""
112
-
113
- path = Path(file_path)
114
- suffix = path.suffix.lower()
115
-
116
- try:
117
- if suffix == ".csv":
118
- df = pd.read_csv(path)
119
- return f"CSV DATA:\n{df.to_csv(index=False)}"
120
-
121
- if suffix in {".xlsx", ".xls"}:
122
- xls = pd.ExcelFile(path)
123
- sections = []
124
-
125
- for sheet in xls.sheet_names:
126
- df = pd.read_excel(path, sheet_name=sheet)
127
- sections.append(f"SHEET {sheet}:\n{df.to_csv(index=False)}")
128
-
129
- return "\n\n".join(sections)
130
-
131
- if suffix in {".py", ".txt", ".md", ".json"}:
132
- return path.read_text(encoding="utf-8", errors="replace")
133
-
134
- if suffix == ".pdf":
135
- reader = PdfReader(path)
136
- return "\n\n".join(
137
- (page.extract_text() or "") for page in reader.pages
138
- )
139
 
140
- return f"Attachment exists at {path.name}, type={suffix}"
 
141
 
142
- except Exception as exc:
143
- return f"Attachment extraction failed: {type(exc).__name__}: {exc}"
144
 
145
 
 
 
 
146
  def looks_reversed(question: str) -> bool:
147
  reversed_text = question[::-1].lower()
148
-
149
- markers = (
150
- " answer",
151
- " write ",
152
- " sentence",
153
- " word ",
154
- " understand ",
155
- )
156
-
157
  return (
158
  sum(m in reversed_text for m in markers)
159
  >= sum(m in question.lower() for m in markers) + 2
160
  )
161
 
162
 
163
- def solve_reversed_text(question: str) -> str | None:
164
  if not looks_reversed(question):
165
  return None
166
-
167
  decoded = question[::-1].lower()
168
-
169
  pairs = {
170
- "left": "right",
171
- "right": "left",
172
- "up": "down",
173
- "down": "up",
174
- "yes": "no",
175
- "no": "yes",
176
- "true": "false",
177
- "false": "true",
178
  }
179
-
180
  if "opposite" not in decoded:
181
  return None
182
-
183
  for source, target in pairs.items():
184
  if re.search(rf"\b{re.escape(source)}\b", decoded):
185
  return target
186
-
187
  return None
188
 
189
 
190
- def solve_commutativity_table(question: str) -> str | None:
191
  if "not commutative" not in question.lower() or "|*|" not in question:
192
  return None
193
 
194
  lines = [line.strip() for line in question.splitlines() if line.strip()]
195
  table_lines = [line for line in lines if line.startswith("|")]
196
-
197
  if len(table_lines) < 3:
198
  return None
199
 
@@ -205,50 +117,37 @@ def solve_commutativity_table(question: str) -> str | None:
205
  cells = [x.strip() for x in row.strip("|").split("|")]
206
  if len(cells) != len(header) + 1:
207
  continue
208
-
209
- row_name = cells[0]
210
  for col_name, value in zip(header, cells[1:]):
211
- matrix[(row_name, col_name)] = value
212
 
213
  involved = set()
214
-
215
  for a in header:
216
  for b in header:
217
  if matrix.get((a, b)) != matrix.get((b, a)):
218
- involved.add(a)
219
- involved.add(b)
220
-
221
- if not involved:
222
- return None
223
 
224
- return ", ".join(sorted(involved))
225
 
226
 
 
 
 
 
 
 
 
 
227
 
228
- def solve_botanical_vegetables(question: str) -> str | None:
229
- """
230
- Deterministic solver for grocery-list questions that explicitly ask for
231
- vegetables while excluding botanical fruits.
232
 
233
- This uses botanical plant-part categories rather than culinary convention:
234
- leaves, stems, roots/tubers, and flower structures qualify; botanical
235
- fruits/seeds/grains/nuts are excluded.
236
- """
237
  lowered = question.lower()
238
-
239
- if (
240
- "botanical fruits" not in lowered
241
- or "vegetables" not in lowered
242
- or "alphabet" not in lowered
243
- or "comma" not in lowered
244
- ):
245
  return None
246
 
247
- # Extract the grocery-list block when present.
248
  match = re.search(
249
  r"(?:list i have so far|list.*?so far)\s*:\s*(.+?)(?:\n\s*\n|i need to)",
250
  question,
251
- flags=re.IGNORECASE | re.DOTALL,
252
  )
253
  if not match:
254
  return None
@@ -258,81 +157,139 @@ def solve_botanical_vegetables(question: str) -> str | None:
258
  for item in match.group(1).replace("\n", " ").split(",")
259
  if item.strip()
260
  ]
 
 
 
261
 
262
- # Common edible vegetative/flower plant parts. This is generic category
263
- # knowledge, not task-ID routing.
264
- botanical_vegetables = {
265
- "artichoke",
266
- "asparagus",
267
- "beet",
268
- "beets",
269
- "bok choy",
270
- "broccoli",
271
- "brussels sprouts",
272
- "cabbage",
273
- "carrot",
274
- "carrots",
275
- "cauliflower",
276
- "celery",
277
- "chard",
278
- "collard greens",
279
- "fresh basil",
280
- "garlic",
281
- "ginger",
282
- "kale",
283
- "leek",
284
- "leeks",
285
- "lettuce",
286
- "onion",
287
- "onions",
288
- "parsnip",
289
- "parsnips",
290
- "potato",
291
- "potatoes",
292
- "radish",
293
- "radishes",
294
- "spinach",
295
- "sweet potato",
296
- "sweet potatoes",
297
- "turnip",
298
- "turnips",
299
  }
 
300
 
301
- selected = [
302
- item
303
- for item in items
304
- if item.lower() in botanical_vegetables
305
- ]
306
 
307
- if not selected:
 
 
 
 
 
 
 
 
 
 
 
308
  return None
309
 
310
- return ", ".join(sorted(selected, key=str.lower))
311
 
 
 
 
 
312
 
313
- def normalize_requested_format(question: str, answer: str) -> str:
314
- """
315
- Apply only formatting transformations explicitly requested by the question.
316
- """
317
- value = clean_answer(answer)
 
 
 
 
318
 
319
- if "without abbreviations" in question.lower():
320
- value = re.sub(
321
- r"\bSt\.\s+",
322
- "Saint ",
323
- value,
324
- flags=re.IGNORECASE,
 
 
 
325
  )
 
 
 
326
 
327
- return value
328
 
329
- def youtube_transcript(question: str) -> str:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
330
  urls = re.findall(
331
  r"https?://(?:www\.)?(?:youtube\.com/[^\s]+|youtu\.be/[^\s]+)",
332
  question,
333
- flags=re.IGNORECASE,
334
  )
335
-
336
  if not urls:
337
  return ""
338
 
@@ -346,7 +303,6 @@ def youtube_transcript(question: str) -> str:
346
 
347
  if parsed.netloc in {"youtu.be", "www.youtu.be"}:
348
  video_id = parsed.path.strip("/").split("/")[0] or None
349
-
350
  elif "youtube.com" in parsed.netloc:
351
  if parsed.path == "/watch":
352
  video_id = parse_qs(parsed.query).get("v", [None])[0]
@@ -363,17 +319,73 @@ def youtube_transcript(question: str) -> str:
363
  text = " ".join(snippet.text for snippet in transcript.snippets)
364
  if text.strip():
365
  sections.append(text)
366
-
367
  except Exception as exc:
368
- print(f"Transcript unavailable for {video_id}: {exc}")
369
 
370
  return "\n\n".join(sections)
371
 
372
 
373
- # ---------------------------------------------------------------------
374
- # Agent
375
- # ---------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
376
 
 
 
 
 
 
 
377
  class ResearchAgent:
378
  def __init__(self):
379
  hf_token = os.getenv("HF_TOKEN")
@@ -383,150 +395,244 @@ class ResearchAgent:
383
  model = InferenceClientModel(
384
  model_id=MODEL_ID,
385
  token=hf_token,
386
- timeout=120,
 
 
387
  )
388
 
 
 
 
 
 
 
389
  self.agent = CodeAgent(
390
- tools=[
391
- DuckDuckGoSearchTool(max_results=6),
392
- VisitWebpageTool(),
393
- ],
394
  model=model,
395
- max_steps=5,
396
  verbosity_level=0,
397
  additional_authorized_imports=[
398
- "math",
399
- "statistics",
400
- "datetime",
401
- "re",
402
  ],
403
  )
404
 
405
- def answer(
406
- self,
407
- question: str,
408
- attachment_content: str = "",
409
- extra_context: str = "",
410
- ) -> str:
411
  prompt = f"""
412
- Solve this GAIA Level-1 task.
413
 
414
  QUESTION:
415
  {question}
416
 
417
- ATTACHMENT CONTENT:
418
- {attachment_content or "No usable attachment content."}
419
 
420
  EXTRA CONTEXT:
421
- {extra_context or "No extra context."}
422
-
423
- Requirements:
424
- - Use named/authoritative sources when the question specifies one.
425
- - Search the web only when needed.
426
- - If transcript or attachment content is supplied, prefer it over guessing.
427
- - Never invent image/audio contents.
428
- - Return ONLY the exact final answer.
429
- - No reasoning, labels, markdown, or explanatory text.
430
- - Respect requested ordering, commas, capitalization, units, and rounding.
 
 
 
 
 
 
431
  """
432
-
433
  result = self.agent.run(prompt)
434
  return clean_answer(str(result))
435
 
436
 
437
- # ---------------------------------------------------------------------
438
- # Evaluation / submission
439
- # ---------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
440
 
441
- def run_and_submit_all(profile: gr.OAuthProfile | None):
442
- if not profile:
443
- return "Please log in to Hugging Face first.", None
 
 
 
444
 
445
- space_id = os.getenv("SPACE_ID")
446
- if not space_id:
447
- return "SPACE_ID is missing.", None
 
 
 
 
448
 
449
- username = str(profile.username)
450
- api_url = DEFAULT_API_URL
451
 
 
 
 
 
452
  try:
453
- agent = ResearchAgent()
454
  except Exception as exc:
455
- return f"Agent init failed: {exc}", None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
456
 
457
  try:
458
- response = requests.get(f"{api_url}/questions", timeout=30)
459
- response.raise_for_status()
460
- questions = response.json()
461
- except Exception as exc:
462
- return f"Could not fetch questions: {exc}", None
463
 
464
- results = []
465
- answers = []
466
- local_solves = 0
 
 
 
 
 
 
467
 
468
- for item in questions:
469
- task_id = item.get("task_id")
470
- question = item.get("question")
471
 
472
- if not task_id or question is None:
473
- continue
 
 
 
 
474
 
 
 
475
  try:
476
- # Generic zero-cost solvers first.
477
- answer = solve_reversed_text(question)
478
-
479
- if answer is None:
480
- answer = solve_commutativity_table(question)
 
 
 
 
 
 
 
 
 
 
 
 
481
 
482
- if answer is None:
483
- answer = solve_botanical_vegetables(question)
484
 
485
- if answer is not None:
486
- local_solves += 1
487
- else:
488
- file_path = download_task_file(api_url, task_id)
489
- attachment = extract_attachment_content(file_path)
490
- transcript = youtube_transcript(question)
491
 
492
- answer = agent.answer(
493
- question=question,
494
- attachment_content=attachment,
495
- extra_context=transcript,
496
- )
497
 
498
- answer = normalize_requested_format(question, answer)
 
499
 
 
 
 
 
500
  except Exception as exc:
501
- print(f"Task {task_id} failed: {exc}")
502
  answer = ""
503
-
504
- answers.append(
505
- {
506
- "task_id": task_id,
507
- "submitted_answer": answer,
 
508
  }
509
- )
510
 
511
- results.append(
512
- {
513
- "Task ID": task_id,
514
- "Question": question,
515
- "Submitted Answer": answer,
516
- }
517
- )
518
 
519
- submission = {
520
- "username": username,
 
 
 
 
521
  "agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
522
  "answers": answers,
523
  }
524
 
525
  try:
526
  response = requests.post(
527
- f"{api_url}/submit",
528
- json=submission,
529
- timeout=90,
530
  )
531
  response.raise_for_status()
532
  data = response.json()
@@ -536,51 +642,109 @@ def run_and_submit_all(profile: gr.OAuthProfile | None):
536
  f"User: {data.get('username')}\n"
537
  f"Overall Score: {data.get('score', 'N/A')}% "
538
  f"({data.get('correct_count', '?')}/{data.get('total_attempted', '?')} correct)\n"
539
- f"Zero-cost deterministic solves: {local_solves}\n"
540
  f"Message: {data.get('message', '')}"
541
  )
542
-
543
- return status, pd.DataFrame(results)
544
 
545
  except Exception as exc:
546
- return (
547
- f"Submission failed: {type(exc).__name__}: {exc}",
548
- pd.DataFrame(results),
549
- )
550
 
551
 
 
 
 
552
  with gr.Blocks() as demo:
553
- gr.Markdown("# GAIA Agent Evaluation Runner — Final")
554
 
555
  gr.Markdown(
556
  """
557
- Final submission build. It uses deterministic solvers first, applies
558
- question-requested formatting normalization, then uses one compact research-agent
559
- pass for unresolved tasks. It does not embed benchmark answer keys.
 
 
 
560
  """
561
  )
562
 
563
  gr.LoginButton()
564
 
565
- run_button = gr.Button("Run Evaluation & Submit All Answers")
 
 
566
 
567
- status_output = gr.Textbox(
568
- label="Run Status / Submission Result",
569
- lines=6,
570
  interactive=False,
571
  )
 
 
 
 
 
 
 
 
 
572
 
573
- results_table = gr.DataFrame(
574
- label="Questions and Agent Answers",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
575
  wrap=True,
576
  )
577
 
578
- run_button.click(
579
- fn=run_and_submit_all,
580
- outputs=[status_output, results_table],
581
  )
582
 
583
 
584
  if __name__ == "__main__":
585
- print("Launching low-cost GAIA agent...")
586
- demo.launch(debug=True, share=False)
 
1
  import os
2
  import re
3
+ import json
4
  import tempfile
5
  from pathlib import Path
6
  from urllib.parse import unquote, urlparse, parse_qs
 
10
  import requests
11
  from pypdf import PdfReader
12
  from youtube_transcript_api import YouTubeTranscriptApi
13
+ from huggingface_hub import InferenceClient, hf_hub_download
14
  from smolagents import (
15
  CodeAgent,
16
  DuckDuckGoSearchTool,
17
  InferenceClientModel,
18
  VisitWebpageTool,
19
+ PythonInterpreterTool,
20
  )
21
 
22
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
23
+ MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen2.5-Coder-32B-Instruct")
24
+ GAIA_DATASET = "gaia-benchmark/GAIA"
25
 
26
 
27
+ # ----------------------------
28
+ # Answer normalization
29
+ # ----------------------------
 
30
  def clean_answer(value: str) -> str:
31
+ value = str(value or "").strip()
32
 
33
+ for prefix in (
34
+ "final answer:",
35
+ "final answer",
36
+ "answer:",
37
+ "answer",
38
+ ):
39
  if value.lower().startswith(prefix):
40
  value = value[len(prefix):].lstrip(": \n\t").strip()
41
  break
42
 
43
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
 
 
 
 
44
  value = value[1:-1].strip()
45
 
46
+ # Remove common wrapper without damaging decimals.
47
+ value = re.sub(r"^\s*FINAL ANSWER\s*:\s*", "", value, flags=re.I)
48
+ if value.endswith(".") and not re.search(r"\d\.\d$", value):
49
+ value = value[:-1].rstrip()
50
+
51
  bad_markers = (
52
  "Thought:",
53
  "Calling tools:",
 
55
  "</code>",
56
  "python_interpreter",
57
  )
58
+ if any(marker.lower() in value.lower() for marker in bad_markers):
59
  return ""
60
 
61
  return value
62
 
63
 
64
+ def normalize_requested_format(question: str, answer: str) -> str:
65
+ value = clean_answer(answer)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
66
 
67
+ if "without abbreviations" in question.lower():
68
+ value = re.sub(r"\bSt\.\s+", "Saint ", value, flags=re.I)
69
 
70
+ return value
 
71
 
72
 
73
+ # ----------------------------
74
+ # Deterministic solvers
75
+ # ----------------------------
76
  def looks_reversed(question: str) -> bool:
77
  reversed_text = question[::-1].lower()
78
+ markers = (" answer", " write ", " sentence", " word ", " understand ")
 
 
 
 
 
 
 
 
79
  return (
80
  sum(m in reversed_text for m in markers)
81
  >= sum(m in question.lower() for m in markers) + 2
82
  )
83
 
84
 
85
+ def solve_reversed_text(question: str):
86
  if not looks_reversed(question):
87
  return None
 
88
  decoded = question[::-1].lower()
 
89
  pairs = {
90
+ "left": "right", "right": "left",
91
+ "up": "down", "down": "up",
92
+ "yes": "no", "no": "yes",
93
+ "true": "false", "false": "true",
 
 
 
 
94
  }
 
95
  if "opposite" not in decoded:
96
  return None
 
97
  for source, target in pairs.items():
98
  if re.search(rf"\b{re.escape(source)}\b", decoded):
99
  return target
 
100
  return None
101
 
102
 
103
+ def solve_commutativity_table(question: str):
104
  if "not commutative" not in question.lower() or "|*|" not in question:
105
  return None
106
 
107
  lines = [line.strip() for line in question.splitlines() if line.strip()]
108
  table_lines = [line for line in lines if line.startswith("|")]
 
109
  if len(table_lines) < 3:
110
  return None
111
 
 
117
  cells = [x.strip() for x in row.strip("|").split("|")]
118
  if len(cells) != len(header) + 1:
119
  continue
 
 
120
  for col_name, value in zip(header, cells[1:]):
121
+ matrix[(cells[0], col_name)] = value
122
 
123
  involved = set()
 
124
  for a in header:
125
  for b in header:
126
  if matrix.get((a, b)) != matrix.get((b, a)):
127
+ involved.update((a, b))
 
 
 
 
128
 
129
+ return ", ".join(sorted(involved)) if involved else None
130
 
131
 
132
+ BOTANICAL_VEGETABLES = {
133
+ "artichoke", "asparagus", "beet", "beets", "bok choy", "broccoli",
134
+ "brussels sprouts", "cabbage", "carrot", "carrots", "cauliflower",
135
+ "celery", "chard", "collard greens", "fresh basil", "garlic", "ginger",
136
+ "kale", "leek", "leeks", "lettuce", "onion", "onions", "parsnip",
137
+ "parsnips", "potato", "potatoes", "radish", "radishes", "spinach",
138
+ "sweet potato", "sweet potatoes", "turnip", "turnips",
139
+ }
140
 
 
 
 
 
141
 
142
+ def solve_botanical_vegetables(question: str):
 
 
 
143
  lowered = question.lower()
144
+ if not all(x in lowered for x in ("botanical fruits", "vegetables", "alphabet", "comma")):
 
 
 
 
 
 
145
  return None
146
 
 
147
  match = re.search(
148
  r"(?:list i have so far|list.*?so far)\s*:\s*(.+?)(?:\n\s*\n|i need to)",
149
  question,
150
+ flags=re.I | re.S,
151
  )
152
  if not match:
153
  return None
 
157
  for item in match.group(1).replace("\n", " ").split(",")
158
  if item.strip()
159
  ]
160
+ selected = [x for x in items if x.lower() in BOTANICAL_VEGETABLES]
161
+ return ", ".join(sorted(selected, key=str.lower)) if selected else None
162
+
163
 
164
+ def deterministic_solve(question: str):
165
+ for name, solver in (
166
+ ("Reversed text", solve_reversed_text),
167
+ ("Commutativity table", solve_commutativity_table),
168
+ ("Botanical vegetables", solve_botanical_vegetables),
169
+ ):
170
+ answer = solver(question)
171
+ if answer is not None:
172
+ return name, normalize_requested_format(question, answer)
173
+ return "LLM required", None
174
+
175
+
176
+ # ----------------------------
177
+ # Attachment handling
178
+ # ----------------------------
179
+ def filename_from_response(response, task_id):
180
+ header = response.headers.get("content-disposition", "")
181
+ match = re.search(r"filename\*=UTF-8''([^;]+)", header, flags=re.I)
182
+ if match:
183
+ return Path(unquote(match.group(1))).name
184
+ match = re.search(r'filename="?([^";]+)"?', header)
185
+ if match:
186
+ return Path(match.group(1).strip()).name
187
+
188
+ content_type = response.headers.get("content-type", "").split(";")[0].lower()
189
+ ext_map = {
190
+ "application/pdf": ".pdf",
191
+ "text/csv": ".csv",
192
+ "application/json": ".json",
193
+ "text/plain": ".txt",
194
+ "audio/mpeg": ".mp3",
195
+ "audio/wav": ".wav",
196
+ "image/png": ".png",
197
+ "image/jpeg": ".jpg",
198
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
199
+ "application/vnd.ms-excel": ".xls",
 
200
  }
201
+ return f"{task_id}{ext_map.get(content_type, '.bin')}"
202
 
 
 
 
 
 
203
 
204
+ def download_scoring_file(task_id: str):
205
+ url = f"{DEFAULT_API_URL}/files/{task_id}"
206
+ try:
207
+ response = requests.get(url, timeout=30)
208
+ if response.status_code != 200 or not response.content:
209
+ return None
210
+ filename = filename_from_response(response, task_id)
211
+ temp_dir = Path(tempfile.mkdtemp(prefix=f"gaia_{task_id[:8]}_"))
212
+ path = temp_dir / filename
213
+ path.write_bytes(response.content)
214
+ return str(path)
215
+ except Exception:
216
  return None
217
 
 
218
 
219
+ def dataset_fallback_file(task_id: str, hf_token: str):
220
+ """Use the gated GAIA validation metadata to locate the exact attachment."""
221
+ try:
222
+ from datasets import load_dataset
223
 
224
+ ds = load_dataset(
225
+ GAIA_DATASET,
226
+ "2023_level1",
227
+ split="validation",
228
+ token=hf_token,
229
+ )
230
+ matches = ds.filter(lambda row: row["task_id"] == task_id)
231
+ if len(matches) == 0:
232
+ return None
233
 
234
+ file_path = matches[0].get("file_path")
235
+ if not file_path:
236
+ return None
237
+
238
+ return hf_hub_download(
239
+ repo_id=GAIA_DATASET,
240
+ repo_type="dataset",
241
+ filename=file_path,
242
+ token=hf_token,
243
  )
244
+ except Exception as exc:
245
+ print(f"GAIA dataset fallback unavailable for {task_id}: {exc}")
246
+ return None
247
 
 
248
 
249
+ def download_task_file(task_id: str, hf_token: str):
250
+ return download_scoring_file(task_id) or dataset_fallback_file(task_id, hf_token)
251
+
252
+
253
+ def extract_attachment_content(file_path):
254
+ if not file_path:
255
+ return ""
256
+
257
+ path = Path(file_path)
258
+ suffix = path.suffix.lower()
259
+
260
+ try:
261
+ if suffix == ".csv":
262
+ return f"CSV DATA:\n{pd.read_csv(path).to_csv(index=False)}"
263
+
264
+ if suffix in {".xlsx", ".xls"}:
265
+ xls = pd.ExcelFile(path)
266
+ sections = []
267
+ for sheet in xls.sheet_names:
268
+ df = pd.read_excel(path, sheet_name=sheet, header=None)
269
+ sections.append(f"SHEET {sheet}:\n{df.to_csv(index=False, header=False)}")
270
+ return "\n\n".join(sections)
271
+
272
+ if suffix in {".py", ".txt", ".md", ".json", ".jsonld"}:
273
+ return path.read_text(encoding="utf-8", errors="replace")
274
+
275
+ if suffix == ".pdf":
276
+ reader = PdfReader(path)
277
+ return "\n\n".join(page.extract_text() or "" for page in reader.pages)
278
+
279
+ return ""
280
+ except Exception as exc:
281
+ return f"Attachment extraction failed: {type(exc).__name__}: {exc}"
282
+
283
+
284
+ # ----------------------------
285
+ # YouTube
286
+ # ----------------------------
287
+ def youtube_transcript(question: str):
288
  urls = re.findall(
289
  r"https?://(?:www\.)?(?:youtube\.com/[^\s]+|youtu\.be/[^\s]+)",
290
  question,
291
+ flags=re.I,
292
  )
 
293
  if not urls:
294
  return ""
295
 
 
303
 
304
  if parsed.netloc in {"youtu.be", "www.youtu.be"}:
305
  video_id = parsed.path.strip("/").split("/")[0] or None
 
306
  elif "youtube.com" in parsed.netloc:
307
  if parsed.path == "/watch":
308
  video_id = parse_qs(parsed.query).get("v", [None])[0]
 
319
  text = " ".join(snippet.text for snippet in transcript.snippets)
320
  if text.strip():
321
  sections.append(text)
 
322
  except Exception as exc:
323
+ print(f"Transcript unavailable: {exc}")
324
 
325
  return "\n\n".join(sections)
326
 
327
 
328
+ # ----------------------------
329
+ # Optional multimodal helpers
330
+ # ----------------------------
331
+ def multimodal_attachment_answer(file_path, question, hf_token):
332
+ """Try HF Inference Providers for image/audio attachments."""
333
+ if not file_path or not hf_token:
334
+ return ""
335
+
336
+ suffix = Path(file_path).suffix.lower()
337
+ client = InferenceClient(token=hf_token)
338
+
339
+ try:
340
+ if suffix in {".png", ".jpg", ".jpeg", ".webp"}:
341
+ # VLM via chat completion. The exact provider/model can be
342
+ # overridden with VISION_MODEL.
343
+ model = os.getenv("VISION_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
344
+ import base64
345
+ mime = {
346
+ ".png": "image/png",
347
+ ".jpg": "image/jpeg",
348
+ ".jpeg": "image/jpeg",
349
+ ".webp": "image/webp",
350
+ }[suffix]
351
+ data = base64.b64encode(Path(file_path).read_bytes()).decode()
352
+ response = client.chat.completions.create(
353
+ model=model,
354
+ messages=[{
355
+ "role": "user",
356
+ "content": [
357
+ {"type": "text", "text": (
358
+ "Answer the question using the attached image. "
359
+ "Return only the answer, with no explanation.\n\n"
360
+ f"Question: {question}"
361
+ )},
362
+ {"type": "image_url", "image_url": {
363
+ "url": f"data:{mime};base64,{data}"
364
+ }},
365
+ ],
366
+ }],
367
+ max_tokens=300,
368
+ )
369
+ return clean_answer(response.choices[0].message.content)
370
+
371
+ if suffix in {".mp3", ".wav", ".m4a", ".flac", ".ogg"}:
372
+ model = os.getenv("AUDIO_MODEL", "openai/whisper-large-v3")
373
+ transcript = client.automatic_speech_recognition(
374
+ file_path,
375
+ model=model,
376
+ )
377
+ text = getattr(transcript, "text", str(transcript))
378
+ return text.strip()
379
+
380
+ except Exception as exc:
381
+ print(f"Multimodal inference unavailable: {exc}")
382
 
383
+ return ""
384
+
385
+
386
+ # ----------------------------
387
+ # Agent
388
+ # ----------------------------
389
  class ResearchAgent:
390
  def __init__(self):
391
  hf_token = os.getenv("HF_TOKEN")
 
395
  model = InferenceClientModel(
396
  model_id=MODEL_ID,
397
  token=hf_token,
398
+ timeout=180,
399
+ temperature=0.1,
400
+ max_tokens=1200,
401
  )
402
 
403
+ tools = [
404
+ DuckDuckGoSearchTool(max_results=8),
405
+ VisitWebpageTool(),
406
+ PythonInterpreterTool(),
407
+ ]
408
+
409
  self.agent = CodeAgent(
410
+ tools=tools,
 
 
 
411
  model=model,
412
+ max_steps=10,
413
  verbosity_level=0,
414
  additional_authorized_imports=[
415
+ "math", "statistics", "datetime", "re",
416
+ "json", "pandas", "numpy",
 
 
417
  ],
418
  )
419
 
420
+ def answer(self, question, attachment_content="", extra_context="", attachment_path=None):
 
 
 
 
 
421
  prompt = f"""
422
+ You are solving ONE GAIA Level-1 benchmark question.
423
 
424
  QUESTION:
425
  {question}
426
 
427
+ ATTACHMENT TEXT:
428
+ {attachment_content or "No text attachment content is available."}
429
 
430
  EXTRA CONTEXT:
431
+ {extra_context or "No YouTube transcript is available."}
432
+
433
+ Rules:
434
+ 1. Solve the question completely. Do not guess.
435
+ 2. Use web search when current/factual/research information is needed.
436
+ 3. Use Python for arithmetic, data processing, tables, dates, and calculations.
437
+ 4. If attachment text is provided, use it as evidence.
438
+ 5. If an attachment is an image/audio file, use the multimodal result supplied separately when available.
439
+ 6. Cross-check important facts before answering.
440
+ 7. Follow every formatting instruction in the question exactly.
441
+ 8. The evaluator uses EXACT MATCH.
442
+ 9. Return ONLY the final answer. No reasoning, no markdown, no labels, no "FINAL ANSWER".
443
+ 10. For numeric answers, do not add units or commas unless the question explicitly requests them.
444
+ 11. For lists, preserve the requested ordering and separator.
445
+
446
+ Work carefully, then output only the answer.
447
  """
 
448
  result = self.agent.run(prompt)
449
  return clean_answer(str(result))
450
 
451
 
452
+ # ----------------------------
453
+ # Question loading
454
+ # ----------------------------
455
+ def get_questions():
456
+ response = requests.get(f"{DEFAULT_API_URL}/questions", timeout=30)
457
+ response.raise_for_status()
458
+ return response.json()
459
+
460
+
461
+ def solve_one(item, agent=None):
462
+ task_id = item.get("task_id")
463
+ question = item.get("question", "")
464
+
465
+ solver, deterministic_answer = deterministic_solve(question)
466
+ if deterministic_answer is not None:
467
+ return {
468
+ "task_id": task_id,
469
+ "question": question,
470
+ "solver": solver,
471
+ "answer": deterministic_answer,
472
+ "attachment": "not needed",
473
+ }
474
+
475
+ hf_token = os.getenv("HF_TOKEN", "")
476
+ file_path = download_task_file(task_id, hf_token)
477
+
478
+ attachment = extract_attachment_content(file_path)
479
+ transcript = youtube_transcript(question)
480
+
481
+ # Give images/audio a dedicated inference attempt before the text agent.
482
+ multimodal = multimodal_attachment_answer(file_path, question, hf_token)
483
+ if multimodal:
484
+ # Audio output is a transcript, not necessarily the answer, so let the
485
+ # main agent consume it as context rather than submitting it directly.
486
+ if Path(file_path).suffix.lower() in {".mp3", ".wav", ".m4a", ".flac", ".ogg"}:
487
+ attachment = f"AUDIO TRANSCRIPT:\n{multimodal}\n\n{attachment}"
488
+ else:
489
+ attachment = f"IMAGE MODEL RESULT:\n{multimodal}\n\n{attachment}"
490
+
491
+ if agent is None:
492
+ agent = ResearchAgent()
493
 
494
+ answer = agent.answer(
495
+ question=question,
496
+ attachment_content=attachment,
497
+ extra_context=transcript,
498
+ attachment_path=file_path,
499
+ )
500
 
501
+ return {
502
+ "task_id": task_id,
503
+ "question": question,
504
+ "solver": "ResearchAgent",
505
+ "answer": normalize_requested_format(question, answer),
506
+ "attachment": Path(file_path).name if file_path else "none",
507
+ }
508
 
 
 
509
 
510
+ # ----------------------------
511
+ # UI actions
512
+ # ----------------------------
513
+ def test_deterministic():
514
  try:
515
+ questions = get_questions()
516
  except Exception as exc:
517
+ return f"Could not fetch questions: {exc}", pd.DataFrame()
518
+
519
+ rows = []
520
+ for item in questions:
521
+ solver, answer = deterministic_solve(item.get("question", ""))
522
+ rows.append({
523
+ "Task ID": item.get("task_id"),
524
+ "Solver": solver,
525
+ "Answer": answer or "",
526
+ "Question": item.get("question", ""),
527
+ })
528
+
529
+ solved = sum(bool(row["Answer"]) for row in rows)
530
+ return (
531
+ f"Deterministic validation complete: {solved}/{len(rows)} "
532
+ "questions matched a local solver. No LLM calls and no submission.",
533
+ pd.DataFrame(rows),
534
+ )
535
+
536
+
537
+ def test_single(task_id):
538
+ task_id = (task_id or "").strip()
539
+ if not task_id:
540
+ return "Enter a Task ID.", pd.DataFrame()
541
 
542
  try:
543
+ questions = get_questions()
544
+ item = next((q for q in questions if q.get("task_id") == task_id), None)
545
+ if item is None:
546
+ return "Task ID not found.", pd.DataFrame()
 
547
 
548
+ result = solve_one(item)
549
+ return (
550
+ f"Test complete. Solver: {result['solver']}\n"
551
+ f"Answer: {result['answer']}\n"
552
+ f"Attachment: {result['attachment']}",
553
+ pd.DataFrame([result]),
554
+ )
555
+ except Exception as exc:
556
+ return f"Single-question test failed: {type(exc).__name__}: {exc}", pd.DataFrame()
557
 
 
 
 
558
 
559
+ def run_all_no_submit():
560
+ try:
561
+ questions = get_questions()
562
+ agent = ResearchAgent()
563
+ except Exception as exc:
564
+ return f"Agent initialization failed: {type(exc).__name__}: {exc}", pd.DataFrame()
565
 
566
+ rows = []
567
+ for index, item in enumerate(questions, start=1):
568
  try:
569
+ result = solve_one(item, agent=agent)
570
+ rows.append(result)
571
+ except Exception as exc:
572
+ rows.append({
573
+ "task_id": item.get("task_id"),
574
+ "question": item.get("question", ""),
575
+ "solver": "ERROR",
576
+ "answer": "",
577
+ "attachment": f"{type(exc).__name__}: {exc}",
578
+ })
579
+
580
+ valid = sum(bool(x["answer"]) for x in rows)
581
+ return (
582
+ f"Dry run complete: {valid}/{len(rows)} produced an answer. "
583
+ "NOT submitted to the leaderboard.",
584
+ pd.DataFrame(rows),
585
+ )
586
 
 
 
587
 
588
+ def submit_all(profile):
589
+ if not profile:
590
+ return "Please log in with Hugging Face first.", None
 
 
 
591
 
592
+ try:
593
+ questions = get_questions()
594
+ agent = ResearchAgent()
595
+ except Exception as exc:
596
+ return f"Agent initialization failed: {type(exc).__name__}: {exc}", None
597
 
598
+ answers = []
599
+ rows = []
600
 
601
+ for item in questions:
602
+ try:
603
+ result = solve_one(item, agent=agent)
604
+ answer = result["answer"]
605
  except Exception as exc:
 
606
  answer = ""
607
+ result = {
608
+ "task_id": item.get("task_id"),
609
+ "question": item.get("question", ""),
610
+ "solver": "ERROR",
611
+ "answer": "",
612
+ "attachment": f"{type(exc).__name__}: {exc}",
613
  }
 
614
 
615
+ answers.append({
616
+ "task_id": item.get("task_id"),
617
+ "submitted_answer": answer,
618
+ })
619
+ rows.append(result)
 
 
620
 
621
+ space_id = os.getenv("SPACE_ID")
622
+ if not space_id:
623
+ return "SPACE_ID is missing from the Space environment.", pd.DataFrame(rows)
624
+
625
+ payload = {
626
+ "username": str(profile.username),
627
  "agent_code": f"https://huggingface.co/spaces/{space_id}/tree/main",
628
  "answers": answers,
629
  }
630
 
631
  try:
632
  response = requests.post(
633
+ f"{DEFAULT_API_URL}/submit",
634
+ json=payload,
635
+ timeout=120,
636
  )
637
  response.raise_for_status()
638
  data = response.json()
 
642
  f"User: {data.get('username')}\n"
643
  f"Overall Score: {data.get('score', 'N/A')}% "
644
  f"({data.get('correct_count', '?')}/{data.get('total_attempted', '?')} correct)\n"
 
645
  f"Message: {data.get('message', '')}"
646
  )
647
+ return status, pd.DataFrame(rows)
 
648
 
649
  except Exception as exc:
650
+ return f"Submission failed: {type(exc).__name__}: {exc}", pd.DataFrame(rows)
 
 
 
651
 
652
 
653
+ # ----------------------------
654
+ # Gradio
655
+ # ----------------------------
656
  with gr.Blocks() as demo:
657
+ gr.Markdown("# GAIA Agent — Test Before Submit")
658
 
659
  gr.Markdown(
660
  """
661
+ This runner separates **validation**, **dry runs**, and **leaderboard submission**.
662
+
663
+ 1. Test deterministic solvers — free, no LLM calls.
664
+ 2. Test one question by Task ID.
665
+ 3. Run all 20 without submitting.
666
+ 4. Only then submit to the GAIA scorer.
667
  """
668
  )
669
 
670
  gr.LoginButton()
671
 
672
+ with gr.Row():
673
+ deterministic_button = gr.Button("1. Test Deterministic Solvers")
674
+ dry_run_button = gr.Button("3. Run All — No Submit")
675
 
676
+ deterministic_status = gr.Textbox(
677
+ label="Deterministic Validation",
678
+ lines=3,
679
  interactive=False,
680
  )
681
+ deterministic_table = gr.DataFrame(
682
+ label="Deterministic Results",
683
+ wrap=True,
684
+ )
685
+
686
+ deterministic_button.click(
687
+ fn=test_deterministic,
688
+ outputs=[deterministic_status, deterministic_table],
689
+ )
690
 
691
+ gr.Markdown("### 2. Test One Question")
692
+ task_id_input = gr.Textbox(
693
+ label="Task ID",
694
+ placeholder="Paste one Task ID from the deterministic table",
695
+ )
696
+ single_button = gr.Button("Run One Question")
697
+ single_status = gr.Textbox(
698
+ label="Single Question Result",
699
+ lines=5,
700
+ interactive=False,
701
+ )
702
+ single_table = gr.DataFrame(
703
+ label="Single Question Details",
704
+ wrap=True,
705
+ )
706
+
707
+ single_button.click(
708
+ fn=test_single,
709
+ inputs=[task_id_input],
710
+ outputs=[single_status, single_table],
711
+ )
712
+
713
+ dry_status = gr.Textbox(
714
+ label="Dry Run Status",
715
+ lines=3,
716
+ interactive=False,
717
+ )
718
+ dry_table = gr.DataFrame(
719
+ label="All Answers — NOT Submitted",
720
+ wrap=True,
721
+ )
722
+
723
+ dry_run_button.click(
724
+ fn=run_all_no_submit,
725
+ outputs=[dry_status, dry_table],
726
+ )
727
+
728
+ gr.Markdown("### 4. Leaderboard Submission")
729
+ submit_button = gr.Button(
730
+ "4. Submit All Answers",
731
+ variant="primary",
732
+ )
733
+ submit_status = gr.Textbox(
734
+ label="Submission Result",
735
+ lines=6,
736
+ interactive=False,
737
+ )
738
+ submit_table = gr.DataFrame(
739
+ label="Submitted Answers",
740
  wrap=True,
741
  )
742
 
743
+ submit_button.click(
744
+ fn=submit_all,
745
+ outputs=[submit_status, submit_table],
746
  )
747
 
748
 
749
  if __name__ == "__main__":
750
+ demo.launch()