akashch1512 commited on
Commit
817e2b1
·
verified ·
1 Parent(s): d3f828e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +572 -346
app.py CHANGED
@@ -5,44 +5,45 @@ import json
5
  import time
6
  import base64
7
  import mimetypes
 
8
  from pathlib import Path
9
- from urllib.parse import urlparse
10
 
11
  import gradio as gr
12
  import pandas as pd
13
  import requests
14
  import spaces
15
- from bs4 import BeautifulSoup
16
  from openai import OpenAI
17
 
18
  # ---------------------------------------------------------------------
19
  # Configuration
20
  # ---------------------------------------------------------------------
21
- DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
 
 
22
 
23
- # Configure these in Hugging Face Space → Settings → Variables and secrets.
24
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
25
  OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o")
26
  OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
27
- AUDIO_MODEL = os.getenv("AUDIO_MODEL", "whisper-1")
 
28
 
29
- MAX_ATTACHMENT_CHARS = 24000
30
- MAX_WEB_CHARS = 14000
31
- HTTP_TIMEOUT = 45
32
 
33
- # ZeroGPU startup compatibility. This function is deliberately never called.
34
- # The agent uses OpenAI's remote API, not the Space GPU.
35
  @spaces.GPU(duration=1)
36
  def _zerogpu_startup_probe():
37
  return None
38
 
39
 
40
  # ---------------------------------------------------------------------
41
- # Utility functions
42
  # ---------------------------------------------------------------------
43
- def clean_answer(text: str) -> str:
44
- """Remove common answer wrappers while preserving the answer itself."""
45
- answer = (text or "").strip()
46
  answer = re.sub(
47
  r"^\s*(?:FINAL ANSWER|FINAL|ANSWER)\s*[:\-]\s*",
48
  "",
@@ -55,23 +56,162 @@ def clean_answer(text: str) -> str:
55
  return answer
56
 
57
 
58
- def extract_urls(text: str) -> list[str]:
59
- urls = re.findall(r"https?://[^\s<>\])}]+", text or "")
60
- return [u.rstrip(".,;:!?") for u in urls]
 
 
61
 
62
 
63
- def image_to_data_url(filename: str, content: bytes) -> str:
64
- mime, _ = mimetypes.guess_type(filename)
65
  if not mime:
66
- mime = "image/jpeg"
67
- encoded = base64.b64encode(content).decode("ascii")
 
68
  return f"data:{mime};base64,{encoded}"
69
 
70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  # ---------------------------------------------------------------------
72
- # GAIA Agent
73
  # ---------------------------------------------------------------------
74
- class BasicAgent:
75
  def __init__(self):
76
  if not OPENAI_API_KEY:
77
  raise ValueError(
@@ -82,210 +222,316 @@ class BasicAgent:
82
  self.client = OpenAI(
83
  api_key=OPENAI_API_KEY,
84
  base_url=OPENAI_BASE_URL,
85
- timeout=120,
86
  max_retries=2,
87
  )
88
- self.api_url = DEFAULT_API_URL
89
- print(f"BasicAgent initialized with model: {OPENAI_MODEL}")
 
 
90
 
91
- def download_attachment(self, task_id: str, file_name: str) -> bytes | None:
92
- if not file_name:
93
- return None
 
 
 
 
 
94
 
95
- url = f"{self.api_url}/files/{task_id}"
96
- print(f"Downloading attachment for task {task_id}: {file_name}")
97
- response = requests.get(url, timeout=HTTP_TIMEOUT)
98
- response.raise_for_status()
99
- return response.content
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
 
101
- def extract_attachment(
102
- self, file_name: str, content: bytes
 
 
103
  ) -> tuple[str, list[dict]]:
104
  """
105
- Return extracted textual context and optional GPT-4o image content.
106
- Handles common GAIA attachment types.
107
  """
108
- suffix = Path(file_name).suffix.lower()
109
- text_parts: list[str] = []
 
 
 
 
 
 
 
110
  image_parts: list[dict] = []
111
 
112
- # Plain text and structured text.
113
- if suffix in {
114
- ".txt", ".md", ".json", ".csv", ".tsv", ".py", ".html",
115
- ".xml", ".log", ".yaml", ".yml",
116
- }:
117
- text_parts.append(
118
- content.decode("utf-8", errors="replace")[:MAX_ATTACHMENT_CHARS]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
  )
120
 
121
- # PDF documents.
122
- elif suffix == ".pdf":
123
- try:
124
- from pypdf import PdfReader
125
- reader = PdfReader(io.BytesIO(content))
126
- pages = []
127
- for page in reader.pages[:35]:
128
- pages.append(page.extract_text() or "")
129
- text_parts.append("\n".join(pages)[:MAX_ATTACHMENT_CHARS])
130
- except Exception as exc:
131
- text_parts.append(f"[PDF extraction error: {exc}]")
 
 
 
 
 
132
 
133
- # Excel workbooks. Include rows, not just a small preview, where possible.
134
- elif suffix in {".xlsx", ".xls"}:
135
- try:
136
- engine = "openpyxl" if suffix == ".xlsx" else "xlrd"
137
- sheets = pd.read_excel(
138
- io.BytesIO(content),
139
- sheet_name=None,
140
- engine=engine,
141
- )
142
- for sheet_name, dataframe in list(sheets.items())[:8]:
143
- text_parts.append(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
144
  f"SHEET: {sheet_name}\n"
145
- f"ROWS: {len(dataframe)}\n"
146
- f"DATA:\n{dataframe.to_csv(index=False)}"
 
147
  )
148
- if sum(len(part) for part in text_parts) > MAX_ATTACHMENT_CHARS:
149
- break
150
- text_parts = ["\n\n".join(text_parts)[:MAX_ATTACHMENT_CHARS]]
151
- except Exception as exc:
152
- text_parts.append(f"[Spreadsheet extraction error: {exc}]")
153
 
154
- # Word documents.
155
- elif suffix == ".docx":
156
- try:
157
  from docx import Document
158
- doc = Document(io.BytesIO(content))
159
- paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
160
  for table in doc.tables:
161
  for row in table.rows:
162
- paragraphs.append(" | ".join(cell.text for cell in row.cells))
163
- text_parts.append("\n".join(paragraphs)[:MAX_ATTACHMENT_CHARS])
164
- except Exception as exc:
165
- text_parts.append(f"[DOCX extraction error: {exc}]")
166
 
167
- # PowerPoint presentations.
168
- elif suffix == ".pptx":
169
- try:
170
  from pptx import Presentation
171
- presentation = Presentation(io.BytesIO(content))
172
- slides_text = []
173
- for slide_number, slide in enumerate(presentation.slides, start=1):
174
- slide_lines = []
175
  for shape in slide.shapes:
176
  if getattr(shape, "has_text_frame", False):
177
- slide_lines.append(shape.text)
178
  if getattr(shape, "has_table", False):
179
  for row in shape.table.rows:
180
- slide_lines.append(
181
  " | ".join(cell.text for cell in row.cells)
182
  )
183
- slides_text.append(
184
- f"SLIDE {slide_number}\n" + "\n".join(slide_lines)
185
- )
186
- text_parts.append("\n\n".join(slides_text)[:MAX_ATTACHMENT_CHARS])
187
- except Exception as exc:
188
- text_parts.append(f"[PPTX extraction error: {exc}]")
189
-
190
- # Images: send directly to GPT-4o vision.
191
- elif suffix in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
192
- image_parts.append({
193
- "type": "image_url",
194
- "image_url": {"url": image_to_data_url(file_name, content)},
195
- })
196
-
197
- # Audio: transcribe through OpenAI's audio transcription endpoint.
198
- elif suffix in {".mp3", ".wav", ".m4a", ".mpeg", ".mpga", ".webm"}:
199
- try:
200
- mime, _ = mimetypes.guess_type(file_name)
201
- with io.BytesIO(content) as audio_file:
202
- audio_file.name = Path(file_name).name
203
- transcription = self.client.audio.transcriptions.create(
204
- model=AUDIO_MODEL,
205
  file=audio_file,
206
  )
207
- text_parts.append(
208
- "AUDIO TRANSCRIPTION:\n" + (transcription.text or "")
209
- )
210
- except Exception as exc:
211
- text_parts.append(f"[Audio transcription error: {exc}]")
212
 
213
- else:
214
- text_parts.append(
215
- f"[Attachment '{file_name}' was downloaded, but its format "
216
- "is not supported by this implementation.]"
217
- )
218
 
219
- return "\n\n".join(text_parts), image_parts
 
220
 
221
- def search_web(self, query: str) -> str:
222
- """Best-effort DuckDuckGo search plus extraction of a few result pages."""
223
- try:
224
- from ddgs import DDGS
225
- results = DDGS().text(query, max_results=5)
226
  except Exception as exc:
227
- print(f"Web search unavailable: {exc}")
228
- return ""
229
-
230
- snippets = []
231
- urls_to_read = []
232
-
233
- for result in results or []:
234
- title = result.get("title", "")
235
- body = result.get("body", "")
236
- url = result.get("href") or result.get("url") or ""
237
- snippets.append(f"TITLE: {title}\nURL: {url}\nSNIPPET: {body}")
238
- if url.startswith("http"):
239
- urls_to_read.append(url)
240
-
241
- page_texts = []
242
- for url in urls_to_read[:2]:
243
- try:
244
- response = requests.get(
245
- url,
246
- timeout=20,
247
- headers={"User-Agent": "Mozilla/5.0 (compatible; GAIAAgent/1.0)"},
248
- )
249
- response.raise_for_status()
250
- content_type = response.headers.get("content-type", "").lower()
251
- if "text/html" not in content_type:
252
- continue
253
- soup = BeautifulSoup(response.text, "html.parser")
254
- for tag in soup(["script", "style", "nav", "footer", "header"]):
255
- tag.decompose()
256
- page_text = soup.get_text(" ", strip=True)
257
- if page_text:
258
- page_texts.append(
259
- f"SOURCE URL: {url}\nPAGE TEXT:\n{page_text[:5000]}"
260
- )
261
- except Exception as exc:
262
- print(f"Could not read search result {url}: {exc}")
263
 
264
- combined = "\n\n".join(snippets + page_texts)
265
- return combined[:MAX_WEB_CHARS]
266
-
267
- def fetch_given_urls(self, question: str) -> str:
268
- """Read URLs explicitly supplied in the question."""
269
- chunks = []
270
- for url in extract_urls(question)[:3]:
271
- try:
272
- response = requests.get(
273
- url,
274
- timeout=25,
275
- headers={"User-Agent": "Mozilla/5.0 (compatible; GAIAAgent/1.0)"},
276
- )
277
- response.raise_for_status()
278
- content_type = response.headers.get("content-type", "").lower()
279
- if "text/html" in content_type or "text/plain" in content_type:
280
- soup = BeautifulSoup(response.text, "html.parser")
281
- for tag in soup(["script", "style", "nav", "footer", "header"]):
282
- tag.decompose()
283
- content = soup.get_text(" ", strip=True)
284
- chunks.append(f"URL: {url}\nCONTENT:\n{content[:5000]}")
285
- except Exception as exc:
286
- chunks.append(f"URL: {url}\nCould not read page: {exc}")
287
- return "\n\n".join(chunks)[:MAX_WEB_CHARS]
288
 
 
289
  def ask_model(
290
  self,
291
  question: str,
@@ -296,40 +542,41 @@ class BasicAgent:
296
  image_parts = image_parts or []
297
 
298
  system_prompt = """
299
- You are a highly capable research and problem-solving assistant answering
300
- GAIA benchmark questions.
301
-
302
- Your objective is to return the exact answer requested by the question.
303
-
304
- Rules:
305
- 1. Analyze the question carefully and identify every constraint.
306
- 2. Use the supplied attachment content when relevant.
307
- 3. Use web evidence when the question asks for obscure, historical, current,
308
- or externally verifiable facts. Prefer reliable primary sources.
309
- 4. For calculations, calculate precisely. For spreadsheet questions, inspect
310
- all relevant rows and distinguish the requested categories.
311
- 5. Do not guess when the supplied evidence contradicts your assumptions.
312
- 6. Follow the requested output format exactly: alphabetize when requested,
313
- use comma-separated values when requested, and use the requested name form.
314
- 7. For numeric answers, do not add units, currency symbols, commas, or
315
- explanatory text unless the question explicitly requests them.
316
- 8. Return ONLY the final answer. No explanation, citations, markdown, or
317
- 'FINAL ANSWER:' prefix.
 
 
318
  """
319
 
320
- user_content = [
321
- {
322
- "type": "text",
323
- "text": (
324
- f"QUESTION:\n{question}\n\n"
325
- f"ATTACHMENT CONTENT:\n"
326
- f"{attachment_context or 'No attachment content.'}\n\n"
327
- f"WEB RESEARCH:\n"
328
- f"{web_context or 'No web research retrieved.'}"
329
- ),
330
- }
331
- ]
332
- user_content.extend(image_parts[:3])
333
 
334
  last_error = None
335
  for attempt in range(3):
@@ -337,214 +584,194 @@ Rules:
337
  response = self.client.chat.completions.create(
338
  model=OPENAI_MODEL,
339
  temperature=0,
340
- max_tokens=700,
341
  messages=[
342
  {"role": "system", "content": system_prompt},
343
  {"role": "user", "content": user_content},
344
  ],
345
  )
346
- answer = response.choices[0].message.content or ""
347
- return clean_answer(answer)
348
  except Exception as exc:
349
  last_error = exc
350
- print(f"OpenAI request attempt {attempt + 1} failed: {exc}")
351
  if attempt < 2:
352
  time.sleep(2 ** attempt)
353
 
354
- raise RuntimeError(f"OpenAI model request failed: {last_error}")
355
 
356
  def __call__(self, task: dict) -> str:
357
  task_id = str(task.get("task_id", ""))
358
  question = str(task.get("question", ""))
359
  file_name = str(task.get("file_name") or "")
360
 
361
- print(f"Processing task {task_id}; attachment={file_name or 'none'}")
362
 
363
- attachment_context = ""
364
- image_parts = []
 
365
 
366
- if file_name:
367
- try:
368
- file_bytes = self.download_attachment(task_id, file_name)
369
- if file_bytes:
370
- attachment_context, image_parts = self.extract_attachment(
371
- file_name, file_bytes
372
- )
373
- except Exception as exc:
374
- attachment_context = f"[Attachment download failed: {exc}]"
375
-
376
- # Search for externally verifiable questions. Avoid unnecessary search
377
- # when the question is clearly about a provided file or simple sorting.
378
- search_context = self.fetch_given_urls(question)
379
- lower_question = question.lower()
380
- needs_research = any(
381
- marker in lower_question
382
- for marker in (
383
- "as of", "who is", "who was", "which", "what year",
384
- "what date", "first name", "last name", "nationality",
385
- "recipient", "according to", "number before", "number after",
386
- "historical", "born", "award", "competition",
387
- )
388
  )
 
 
 
 
389
 
390
- if needs_research and not attachment_context:
391
- query = question[:450]
392
- found = self.search_web(query)
393
- if found:
394
- search_context = (search_context + "\n\n" + found).strip()
395
 
396
  answer = self.ask_model(
397
  question=question,
398
- attachment_context=attachment_context,
399
- web_context=search_context,
400
  image_parts=image_parts,
401
  )
402
-
403
- print(f"Answer for {task_id}: {answer}")
404
  return answer
405
 
406
 
407
  # ---------------------------------------------------------------------
408
- # Course evaluation / submission workflow
409
  # ---------------------------------------------------------------------
410
  def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
411
- space_id = os.getenv("SPACE_ID", "")
412
-
413
  if not profile:
414
- return "Please sign in to Hugging Face using the login button.", pd.DataFrame()
415
 
416
  username = str(profile.username).strip()
417
- questions_url = f"{DEFAULT_API_URL}/questions"
418
- submit_url = f"{DEFAULT_API_URL}/submit"
419
 
420
  try:
421
- agent = BasicAgent()
422
  except Exception as exc:
423
  return f"Agent initialization failed: {exc}", pd.DataFrame()
424
 
425
- agent_code = (
426
- f"https://huggingface.co/spaces/{space_id}/tree/main"
427
- if space_id else ""
428
- )
429
-
430
  try:
431
  response = requests.get(questions_url, timeout=HTTP_TIMEOUT)
432
  response.raise_for_status()
433
- questions_data = response.json()
434
- if not isinstance(questions_data, list) or not questions_data:
435
- return "The scoring API returned no questions.", pd.DataFrame()
436
  except Exception as exc:
437
- return f"Could not fetch benchmark questions: {exc}", pd.DataFrame()
438
 
439
- results_log = []
440
- answers_payload = []
441
- total = len(questions_data)
442
 
443
- for index, item in enumerate(questions_data, start=1):
444
- task_id = item.get("task_id")
445
- question_text = item.get("question")
446
 
447
- if not task_id or question_text is None:
 
 
 
448
  continue
449
 
450
- progress((index - 1) / max(total, 1), desc=f"Solving question {index}/{total}")
 
 
 
451
 
452
  try:
453
- answer = agent(item)
454
- if not answer:
455
- answer = "Unable to determine"
456
-
457
- answers_payload.append({
458
- "task_id": task_id,
459
- "submitted_answer": answer,
460
- })
461
- display_answer = answer
462
-
463
  except Exception as exc:
464
  print(f"Task {task_id} failed: {exc}")
 
465
  display_answer = f"AGENT ERROR: {exc}"
466
- # Include a placeholder so every task is represented in the submission.
467
- answers_payload.append({
468
- "task_id": task_id,
469
- "submitted_answer": "Unable to determine",
470
- })
471
 
472
- results_log.append({
 
 
 
 
473
  "Task ID": task_id,
474
- "Question": question_text,
475
- "File": item.get("file_name", ""),
476
  "Submitted Answer": display_answer,
477
  })
478
 
479
- results_df = pd.DataFrame(results_log)
 
 
480
 
481
- if not answers_payload:
482
- return "No answers were generated; nothing was submitted.", results_df
483
-
484
- submission_data = {
 
 
485
  "username": username,
486
  "agent_code": agent_code,
487
- "answers": answers_payload,
488
  }
489
 
490
- progress(0.95, desc="Submitting answers for scoring")
491
-
492
  try:
493
  response = requests.post(
494
  submit_url,
495
- json=submission_data,
496
- timeout=120,
497
  )
498
  response.raise_for_status()
499
  result = response.json()
500
-
501
- final_status = (
502
  "Submission successful!\n"
503
  f"User: {result.get('username', username)}\n"
504
  f"Score: {result.get('score', 'N/A')}% "
505
  f"({result.get('correct_count', '?')}/"
506
- f"{result.get('total_attempted', len(answers_payload))} correct)\n"
507
- f"Message: {result.get('message', 'No message received.')}"
508
  )
509
  progress(1.0, desc="Finished")
510
- return final_status, results_df
511
-
512
  except requests.HTTPError as exc:
513
- detail = f"HTTP {exc.response.status_code}"
514
- try:
515
- error_json = exc.response.json()
516
- detail += f": {error_json.get('detail', exc.response.text[:500])}"
517
- except Exception:
518
- detail += f": {exc.response.text[:500]}"
519
- return f"Submission failed: {detail}", results_df
520
-
521
- except requests.RequestException as exc:
522
- return f"Submission network error: {exc}", results_df
523
-
524
  except Exception as exc:
525
- return f"Unexpected submission error: {exc}", results_df
526
 
527
 
528
  # ---------------------------------------------------------------------
529
- # Gradio interface
530
  # ---------------------------------------------------------------------
531
  with gr.Blocks() as demo:
532
  gr.Markdown("# GAIA Final Assignment — GPT-4o Agent")
533
  gr.Markdown(
534
  """
535
- This agent uses OpenAI GPT-4o, web search, and attachment processing.
 
 
 
 
 
 
536
 
537
- **Before running:**
538
- 1. Add `OPENAI_API_KEY` under Space Settings → Variables and secrets.
539
- 2. Optionally set `OPENAI_MODEL` to `gpt-4o` (the default).
540
- 3. Sign in to Hugging Face below.
541
- 4. Click the evaluation button and wait for the score.
542
 
543
- The OpenAI API is billed separately from a ChatGPT subscription.
544
- This app uses remote OpenAI inference; the ZeroGPU probe is not called.
545
  """
546
  )
547
-
548
  gr.LoginButton()
549
  run_button = gr.Button(
550
  "Run Evaluation & Submit All Answers",
@@ -559,7 +786,6 @@ This app uses remote OpenAI inference; the ZeroGPU probe is not called.
559
  label="Questions and Agent Answers",
560
  wrap=True,
561
  )
562
-
563
  run_button.click(
564
  fn=run_and_submit_all,
565
  outputs=[status_output, results_table],
@@ -567,7 +793,7 @@ This app uses remote OpenAI inference; the ZeroGPU probe is not called.
567
 
568
 
569
  if __name__ == "__main__":
570
- print("Starting GAIA GPT-4o evaluation app...")
571
  print(f"Model: {OPENAI_MODEL}")
572
  print(f"Space ID: {os.getenv('SPACE_ID', 'local')}")
573
- demo.launch(debug=True, share=False)
 
5
  import time
6
  import base64
7
  import mimetypes
8
+ import tempfile
9
  from pathlib import Path
10
+ from typing import Any
11
 
12
  import gradio as gr
13
  import pandas as pd
14
  import requests
15
  import spaces
 
16
  from openai import OpenAI
17
 
18
  # ---------------------------------------------------------------------
19
  # Configuration
20
  # ---------------------------------------------------------------------
21
+ SCORING_API = "https://agents-course-unit4-scoring.hf.space"
22
+ GAIA_DATASET_ID = "gaia-benchmark/GAIA"
23
+ GAIA_CONFIG = "2023_level1"
24
+ GAIA_SPLIT = "validation"
25
 
 
26
  OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
27
  OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o")
28
  OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
29
+ OPENAI_AUDIO_MODEL = os.getenv("OPENAI_AUDIO_MODEL", "whisper-1")
30
+ HF_TOKEN = os.getenv("HF_TOKEN", "")
31
 
32
+ MAX_CONTEXT_CHARS = 30000
33
+ MAX_SEARCH_CHARS = 16000
34
+ HTTP_TIMEOUT = 40
35
 
36
+ # ZeroGPU startup compatibility. Never called: model inference is remote.
 
37
  @spaces.GPU(duration=1)
38
  def _zerogpu_startup_probe():
39
  return None
40
 
41
 
42
  # ---------------------------------------------------------------------
43
+ # General utilities
44
  # ---------------------------------------------------------------------
45
+ def clean_answer(value: Any) -> str:
46
+ answer = str(value or "").strip()
 
47
  answer = re.sub(
48
  r"^\s*(?:FINAL ANSWER|FINAL|ANSWER)\s*[:\-]\s*",
49
  "",
 
56
  return answer
57
 
58
 
59
+ def get_urls(text: str) -> list[str]:
60
+ return [
61
+ value.rstrip(".,;:!?)]}")
62
+ for value in re.findall(r"https?://[^\s<>]+", text or "")
63
+ ]
64
 
65
 
66
+ def file_to_data_url(path: str) -> str:
67
+ mime, _ = mimetypes.guess_type(path)
68
  if not mime:
69
+ mime = "application/octet-stream"
70
+ with open(path, "rb") as handle:
71
+ encoded = base64.b64encode(handle.read()).decode("ascii")
72
  return f"data:{mime};base64,{encoded}"
73
 
74
 
75
+ def parse_vtt(path: str, max_chars: int = 16000) -> str:
76
+ """Basic WebVTT parser; strips timestamps, cue numbers, and duplicate lines."""
77
+ try:
78
+ raw = Path(path).read_text(encoding="utf-8", errors="replace")
79
+ except Exception:
80
+ return ""
81
+
82
+ lines = []
83
+ seen = set()
84
+ for line in raw.splitlines():
85
+ line = line.strip()
86
+ if (
87
+ not line
88
+ or line == "WEBVTT"
89
+ or "-->" in line
90
+ or line.isdigit()
91
+ or line.startswith(("NOTE", "Kind:", "Language:"))
92
+ ):
93
+ continue
94
+ line = re.sub(r"<[^>]+>", "", line)
95
+ line = re.sub(r"&amp;", "&", line)
96
+ line = re.sub(r"&quot;", '"', line)
97
+ if line and line not in seen:
98
+ seen.add(line)
99
+ lines.append(line)
100
+ return "\n".join(lines)[:max_chars]
101
+
102
+
103
+ # ---------------------------------------------------------------------
104
+ # GAIA attachments
105
+ # ---------------------------------------------------------------------
106
+ class GaiaAttachmentResolver:
107
+ """
108
+ Resolve a task_id to GAIA's repository-relative file_path.
109
+
110
+ The course scoring Space's /files/{task_id} endpoint has been reported
111
+ to return 404 because its file-path map is not populated for the current
112
+ GAIA dataset layout. This resolver bypasses that endpoint.
113
+ """
114
+
115
+ def __init__(self):
116
+ self.path_by_task_id: dict[str, str] = {}
117
+ self.initialized = False
118
+
119
+ def initialize(self):
120
+ if self.initialized:
121
+ return
122
+ self.initialized = True
123
+
124
+ if not HF_TOKEN:
125
+ print(
126
+ "HF_TOKEN is not configured. GAIA dataset attachments "
127
+ "cannot be downloaded directly."
128
+ )
129
+ return
130
+
131
+ try:
132
+ from datasets import load_dataset
133
+
134
+ print(
135
+ f"Loading GAIA metadata: {GAIA_DATASET_ID}, "
136
+ f"{GAIA_CONFIG}, {GAIA_SPLIT}"
137
+ )
138
+ dataset = load_dataset(
139
+ GAIA_DATASET_ID,
140
+ GAIA_CONFIG,
141
+ split=GAIA_SPLIT,
142
+ token=HF_TOKEN,
143
+ )
144
+
145
+ for row in dataset:
146
+ task_id = row.get("task_id")
147
+ file_path = row.get("file_path")
148
+ if task_id and file_path:
149
+ self.path_by_task_id[str(task_id)] = str(file_path)
150
+
151
+ print(
152
+ f"Loaded {len(self.path_by_task_id)} GAIA task-to-file mappings."
153
+ )
154
+ except Exception as exc:
155
+ print(f"Could not load GAIA metadata: {exc}")
156
+ print(
157
+ "Check that you accepted the GAIA dataset conditions and "
158
+ "that HF_TOKEN has read access."
159
+ )
160
+
161
+ def download(self, task_id: str, file_name: str) -> str | None:
162
+ if not file_name:
163
+ return None
164
+
165
+ # 1. Try the course attachment endpoint first.
166
+ try:
167
+ response = requests.get(
168
+ f"{SCORING_API}/files/{task_id}",
169
+ timeout=HTTP_TIMEOUT,
170
+ )
171
+ if response.ok and response.content:
172
+ suffix = Path(file_name).suffix
173
+ fd, path = tempfile.mkstemp(suffix=suffix)
174
+ os.close(fd)
175
+ Path(path).write_bytes(response.content)
176
+ print(f"Attachment retrieved from course API: {task_id}")
177
+ return path
178
+ print(
179
+ f"Course /files endpoint returned {response.status_code} "
180
+ f"for {task_id}; trying GAIA dataset fallback."
181
+ )
182
+ except requests.RequestException as exc:
183
+ print(f"Course file endpoint failed for {task_id}: {exc}")
184
+
185
+ # 2. Resolve and download from the gated GAIA dataset.
186
+ self.initialize()
187
+ relative_path = self.path_by_task_id.get(str(task_id))
188
+ if not relative_path:
189
+ print(
190
+ f"No file_path mapping for task {task_id}. "
191
+ "Check GAIA dataset access and validation metadata."
192
+ )
193
+ return None
194
+
195
+ try:
196
+ from huggingface_hub import hf_hub_download
197
+
198
+ local_path = hf_hub_download(
199
+ repo_id=GAIA_DATASET_ID,
200
+ filename=relative_path,
201
+ repo_type="dataset",
202
+ token=HF_TOKEN,
203
+ )
204
+ print(f"Downloaded GAIA attachment: {relative_path}")
205
+ return local_path
206
+ except Exception as exc:
207
+ print(f"GAIA attachment download failed for {task_id}: {exc}")
208
+ return None
209
+
210
+
211
  # ---------------------------------------------------------------------
212
+ # Main agent
213
  # ---------------------------------------------------------------------
214
+ class GaiaAgent:
215
  def __init__(self):
216
  if not OPENAI_API_KEY:
217
  raise ValueError(
 
222
  self.client = OpenAI(
223
  api_key=OPENAI_API_KEY,
224
  base_url=OPENAI_BASE_URL,
225
+ timeout=150,
226
  max_retries=2,
227
  )
228
+ self.attachments = GaiaAttachmentResolver()
229
+ self.http = requests.Session()
230
+ self.http.headers.update({"User-Agent": "Mozilla/5.0 GAIA-Course-Agent/2.0"})
231
+ print(f"Initialized {OPENAI_MODEL} agent.")
232
 
233
+ # -------------------- Web research --------------------
234
+ def search_web(self, query: str) -> str:
235
+ try:
236
+ from ddgs import DDGS
237
+ results = DDGS().text(query, max_results=6)
238
+ except Exception as exc:
239
+ print(f"Search failed: {exc}")
240
+ return ""
241
 
242
+ chunks = []
243
+ urls = []
244
+ for result in results or []:
245
+ title = result.get("title", "")
246
+ body = result.get("body", "")
247
+ url = result.get("href") or result.get("url") or ""
248
+ chunks.append(f"TITLE: {title}\nURL: {url}\nSNIPPET: {body}")
249
+ if url.startswith("http"):
250
+ urls.append(url)
251
+
252
+ # Read a few source pages for evidence beyond snippets.
253
+ for url in urls[:3]:
254
+ try:
255
+ response = self.http.get(url, timeout=20)
256
+ response.raise_for_status()
257
+ content_type = response.headers.get("content-type", "").lower()
258
+ if "html" not in content_type:
259
+ continue
260
+ from bs4 import BeautifulSoup
261
+ soup = BeautifulSoup(response.text, "html.parser")
262
+ for tag in soup(["script", "style", "nav", "footer", "header"]):
263
+ tag.decompose()
264
+ page = soup.get_text(" ", strip=True)
265
+ if page:
266
+ chunks.append(f"PAGE URL: {url}\nPAGE CONTENT: {page[:3500]}")
267
+ except Exception as exc:
268
+ print(f"Could not read {url}: {exc}")
269
+
270
+ return "\n\n".join(chunks)[:MAX_SEARCH_CHARS]
271
+
272
+ def read_question_urls(self, question: str) -> str:
273
+ chunks = []
274
+ for url in get_urls(question)[:3]:
275
+ # YouTube is handled separately.
276
+ if "youtube.com" in url or "youtu.be" in url:
277
+ continue
278
+ try:
279
+ response = self.http.get(url, timeout=25)
280
+ response.raise_for_status()
281
+ content_type = response.headers.get("content-type", "").lower()
282
+ if "html" not in content_type and "text" not in content_type:
283
+ continue
284
+ from bs4 import BeautifulSoup
285
+ soup = BeautifulSoup(response.text, "html.parser")
286
+ for tag in soup(["script", "style", "nav", "footer", "header"]):
287
+ tag.decompose()
288
+ chunks.append(f"URL: {url}\nCONTENT:\n{soup.get_text(' ', strip=True)[:6000]}")
289
+ except Exception as exc:
290
+ chunks.append(f"URL: {url}\n[Page retrieval failed: {exc}]")
291
+ return "\n\n".join(chunks)[:MAX_SEARCH_CHARS]
292
 
293
+ # -------------------- YouTube retrieval --------------------
294
+ def youtube_context(
295
+ self,
296
+ question: str,
297
  ) -> tuple[str, list[dict]]:
298
  """
299
+ Best-effort YouTube transcript retrieval and visual frame sampling.
300
+ Requires yt-dlp and opencv-python-headless.
301
  """
302
+ youtube_urls = [
303
+ url for url in get_urls(question)
304
+ if "youtube.com" in url or "youtu.be" in url
305
+ ]
306
+ if not youtube_urls:
307
+ return "", []
308
+
309
+ url = youtube_urls[0]
310
+ transcript = ""
311
  image_parts: list[dict] = []
312
 
313
+ try:
314
+ import yt_dlp
315
+
316
+ with tempfile.TemporaryDirectory() as temp_dir:
317
+ subtitle_opts = {
318
+ "skip_download": True,
319
+ "writesubtitles": True,
320
+ "writeautomaticsub": True,
321
+ "subtitleslangs": ["en", "en-US", "en-GB"],
322
+ "subtitlesformat": "vtt",
323
+ "outtmpl": str(Path(temp_dir) / "yt.%(ext)s"),
324
+ "quiet": True,
325
+ "no_warnings": True,
326
+ "noplaylist": True,
327
+ "socket_timeout": 20,
328
+ }
329
+ with yt_dlp.YoutubeDL(subtitle_opts) as ydl:
330
+ info = ydl.extract_info(url, download=True)
331
+
332
+ subtitle_files = list(Path(temp_dir).glob("*.vtt"))
333
+ if subtitle_files:
334
+ transcript = parse_vtt(str(subtitle_files[0]))
335
+
336
+ title = (info or {}).get("title", "")
337
+ duration = (info or {}).get("duration", 0)
338
+ transcript_context = (
339
+ f"YouTube title: {title}\n"
340
+ f"Duration seconds: {duration}\n"
341
+ f"Transcript:\n{transcript}"
342
+ )
343
+
344
+ # Only download and sample frames for explicitly visual questions.
345
+ q = question.lower()
346
+ visual_task = any(
347
+ term in q for term in (
348
+ "on camera", "on screen", "shown", "visible",
349
+ "visually", "bird species", "what color", "how many",
350
+ "appears in the video", "in the scene",
351
+ )
352
+ )
353
+
354
+ if visual_task:
355
+ video_opts = {
356
+ "format": "worst[ext=mp4]/worst",
357
+ "outtmpl": str(Path(temp_dir) / "video.%(ext)s"),
358
+ "quiet": True,
359
+ "no_warnings": True,
360
+ "noplaylist": True,
361
+ "socket_timeout": 25,
362
+ "max_filesize": 100 * 1024 * 1024,
363
+ }
364
+ try:
365
+ with yt_dlp.YoutubeDL(video_opts) as ydl:
366
+ ydl.download([url])
367
+
368
+ video_files = [
369
+ p for p in Path(temp_dir).glob("video.*")
370
+ if p.suffix.lower() in {".mp4", ".mkv", ".webm"}
371
+ ]
372
+ if video_files:
373
+ import cv2
374
+ from PIL import Image
375
+
376
+ cap = cv2.VideoCapture(str(video_files[0]))
377
+ frame_count = cap.get(cv2.CAP_PROP_FRAME_COUNT)
378
+ fps = cap.get(cv2.CAP_PROP_FPS) or 25
379
+ duration_seconds = (
380
+ frame_count / fps if fps > 0 else 0
381
+ )
382
+
383
+ if duration_seconds > 0:
384
+ # Sample up to 18 frames spread over the video.
385
+ sample_count = min(
386
+ 18, max(6, int(duration_seconds / 8))
387
+ )
388
+ timestamps = [
389
+ duration_seconds * i / max(sample_count - 1, 1)
390
+ for i in range(sample_count)
391
+ ]
392
+ for timestamp in timestamps:
393
+ cap.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000)
394
+ ok, frame = cap.read()
395
+ if not ok:
396
+ continue
397
+ frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
398
+ image = Image.fromarray(frame)
399
+ image.thumbnail((700, 450))
400
+ buffer = io.BytesIO()
401
+ image.save(buffer, format="JPEG", quality=78)
402
+ encoded = base64.b64encode(
403
+ buffer.getvalue()
404
+ ).decode("ascii")
405
+ image_parts.append({
406
+ "type": "image_url",
407
+ "image_url": {
408
+ "url": f"data:image/jpeg;base64,{encoded}",
409
+ "detail": "high",
410
+ },
411
+ })
412
+ cap.release()
413
+ except Exception as exc:
414
+ print(f"YouTube frame sampling failed: {exc}")
415
+
416
+ return transcript_context, image_parts
417
+
418
+ except Exception as exc:
419
+ print(f"YouTube retrieval failed: {exc}")
420
+ return (
421
+ "The YouTube video could not be retrieved automatically. "
422
+ f"Retrieval error: {exc}",
423
+ [],
424
  )
425
 
426
+ # -------------------- Attachment processing --------------------
427
+ def process_attachment(
428
+ self,
429
+ task_id: str,
430
+ file_name: str,
431
+ ) -> tuple[str, list[dict]]:
432
+ if not file_name:
433
+ return "", []
434
+
435
+ path = self.attachments.download(task_id, file_name)
436
+ if not path:
437
+ return (
438
+ f"Attachment '{file_name}' could not be retrieved. "
439
+ "Do not pretend to have inspected it.",
440
+ [],
441
+ )
442
 
443
+ suffix = Path(file_name).suffix.lower()
444
+ text = ""
445
+ images: list[dict] = []
446
+
447
+ try:
448
+ if suffix in {
449
+ ".txt", ".md", ".json", ".csv", ".tsv", ".py",
450
+ ".html", ".xml", ".log", ".yaml", ".yml",
451
+ }:
452
+ text = Path(path).read_text(
453
+ encoding="utf-8", errors="replace"
454
+ )[:MAX_CONTEXT_CHARS]
455
+
456
+ elif suffix == ".pdf":
457
+ from pypdf import PdfReader
458
+ reader = PdfReader(path)
459
+ pages = [
460
+ page.extract_text() or ""
461
+ for page in reader.pages[:40]
462
+ ]
463
+ text = "\n".join(pages)[:MAX_CONTEXT_CHARS]
464
+
465
+ elif suffix in {".xlsx", ".xls"}:
466
+ sheets = pd.read_excel(path, sheet_name=None)
467
+ chunks = []
468
+ for sheet_name, frame in sheets.items():
469
+ chunks.append(
470
  f"SHEET: {sheet_name}\n"
471
+ f"ROWS: {len(frame)}\n"
472
+ f"COLUMNS: {', '.join(map(str, frame.columns))}\n"
473
+ f"DATA:\n{frame.to_csv(index=False)}"
474
  )
475
+ text = "\n\n".join(chunks)[:MAX_CONTEXT_CHARS]
 
 
 
 
476
 
477
+ elif suffix == ".docx":
 
 
478
  from docx import Document
479
+ doc = Document(path)
480
+ lines = [p.text for p in doc.paragraphs if p.text.strip()]
481
  for table in doc.tables:
482
  for row in table.rows:
483
+ lines.append(" | ".join(c.text for c in row.cells))
484
+ text = "\n".join(lines)[:MAX_CONTEXT_CHARS]
 
 
485
 
486
+ elif suffix == ".pptx":
 
 
487
  from pptx import Presentation
488
+ prs = Presentation(path)
489
+ slides = []
490
+ for number, slide in enumerate(prs.slides, start=1):
491
+ parts = []
492
  for shape in slide.shapes:
493
  if getattr(shape, "has_text_frame", False):
494
+ parts.append(shape.text)
495
  if getattr(shape, "has_table", False):
496
  for row in shape.table.rows:
497
+ parts.append(
498
  " | ".join(cell.text for cell in row.cells)
499
  )
500
+ slides.append(f"SLIDE {number}\n" + "\n".join(parts))
501
+ text = "\n\n".join(slides)[:MAX_CONTEXT_CHARS]
502
+
503
+ elif suffix in {".png", ".jpg", ".jpeg", ".webp", ".bmp"}:
504
+ images.append({
505
+ "type": "image_url",
506
+ "image_url": {
507
+ "url": file_to_data_url(path),
508
+ "detail": "high",
509
+ },
510
+ })
511
+
512
+ elif suffix in {".mp3", ".wav", ".m4a", ".mpeg", ".mpga", ".webm"}:
513
+ with open(path, "rb") as audio_file:
514
+ response = self.client.audio.transcriptions.create(
515
+ model=OPENAI_AUDIO_MODEL,
 
 
 
 
 
 
516
  file=audio_file,
517
  )
518
+ text = "AUDIO TRANSCRIPT:\n" + (response.text or "")
 
 
 
 
519
 
520
+ elif suffix in {".mp4", ".mov", ".mkv"}:
521
+ text = (
522
+ "A video attachment was found. Automatic video attachment "
523
+ "frame extraction is not configured for this file type."
524
+ )
525
 
526
+ else:
527
+ text = f"Unsupported attachment extension: {suffix}"
528
 
 
 
 
 
 
529
  except Exception as exc:
530
+ text = f"Attachment processing failed for {file_name}: {exc}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
531
 
532
+ return text, images
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
533
 
534
+ # -------------------- Model call --------------------
535
  def ask_model(
536
  self,
537
  question: str,
 
542
  image_parts = image_parts or []
543
 
544
  system_prompt = """
545
+ You are a meticulous GAIA benchmark solver. Your score depends on returning
546
+ the exact ground-truth answer, not on giving a helpful explanation.
547
+
548
+ Work rules:
549
+ - Read the entire question and obey every constraint.
550
+ - Treat attachment content and retrieved webpage content as evidence.
551
+ - For questions about specific people, awards, jersey numbers, dates, or
552
+ historical facts, verify with reliable sources and cross-check.
553
+ - For spreadsheet questions, inspect all rows and calculate exactly.
554
+ - For image/video questions, inspect the provided images/frames. Do not claim
555
+ you cannot view them if image content has actually been supplied.
556
+ - For audio questions, use the transcript. Do not answer from assumptions.
557
+ - For code questions, reason through the code carefully. If possible, calculate
558
+ the exact output, including types, indexing, and floating-point behavior.
559
+ - For botanical classification, distinguish botanical fruits from roots,
560
+ leaves, stems, flowers, and other non-fruit plant parts. Do not simply use
561
+ culinary categories.
562
+ - Never invent facts or claim to have opened an attachment that was not supplied.
563
+ - Follow the requested output format precisely.
564
+ - Return only the final answer. No explanation, no citations, no markdown,
565
+ and do not include 'FINAL ANSWER:'.
566
  """
567
 
568
+ user_content = [{
569
+ "type": "text",
570
+ "text": (
571
+ f"QUESTION:\n{question}\n\n"
572
+ f"ATTACHMENT CONTENT:\n"
573
+ f"{attachment_context or 'No attachment content was provided.'}\n\n"
574
+ f"WEB RESEARCH:\n"
575
+ f"{web_context or 'No web research was retrieved.'}\n\n"
576
+ "Now solve the question. Output only the requested answer."
577
+ ),
578
+ }]
579
+ user_content.extend(image_parts[:20])
 
580
 
581
  last_error = None
582
  for attempt in range(3):
 
584
  response = self.client.chat.completions.create(
585
  model=OPENAI_MODEL,
586
  temperature=0,
587
+ max_tokens=900,
588
  messages=[
589
  {"role": "system", "content": system_prompt},
590
  {"role": "user", "content": user_content},
591
  ],
592
  )
593
+ return clean_answer(response.choices[0].message.content or "")
 
594
  except Exception as exc:
595
  last_error = exc
596
+ print(f"OpenAI attempt {attempt + 1} failed: {exc}")
597
  if attempt < 2:
598
  time.sleep(2 ** attempt)
599
 
600
+ raise RuntimeError(f"OpenAI request failed: {last_error}")
601
 
602
  def __call__(self, task: dict) -> str:
603
  task_id = str(task.get("task_id", ""))
604
  question = str(task.get("question", ""))
605
  file_name = str(task.get("file_name") or "")
606
 
607
+ print(f"Task {task_id}; attachment={file_name or 'none'}")
608
 
609
+ attachment_context, attachment_images = self.process_attachment(
610
+ task_id, file_name
611
+ )
612
 
613
+ web_context = self.read_question_urls(question)
614
+ youtube_context, youtube_images = self.youtube_context(question)
615
+
616
+ # Search external facts if no explicit URL already supplied.
617
+ lower = question.lower()
618
+ research_markers = (
619
+ "as of", "who is", "who was", "which person", "what year",
620
+ "what date", "first name", "last name", "nationality",
621
+ "recipient", "number before", "number after", "award",
622
+ "competition", "according to", "published between",
 
 
 
 
 
 
 
 
 
 
 
 
623
  )
624
+ if any(marker in lower for marker in research_markers):
625
+ search = self.search_web(question[:500])
626
+ if search:
627
+ web_context = (web_context + "\n\n" + search).strip()
628
 
629
+ combined_context = "\n\n".join(
630
+ part for part in (attachment_context, youtube_context) if part
631
+ )
632
+ image_parts = attachment_images + youtube_images
 
633
 
634
  answer = self.ask_model(
635
  question=question,
636
+ attachment_context=combined_context,
637
+ web_context=web_context,
638
  image_parts=image_parts,
639
  )
640
+ print(f"Answer [{task_id}]: {answer}")
 
641
  return answer
642
 
643
 
644
  # ---------------------------------------------------------------------
645
+ # Evaluation and submission
646
  # ---------------------------------------------------------------------
647
  def run_and_submit_all(profile: gr.OAuthProfile | None, progress=gr.Progress()):
 
 
648
  if not profile:
649
+ return "Please sign in to Hugging Face first.", pd.DataFrame()
650
 
651
  username = str(profile.username).strip()
652
+ questions_url = f"{SCORING_API}/questions"
653
+ submit_url = f"{SCORING_API}/submit"
654
 
655
  try:
656
+ agent = GaiaAgent()
657
  except Exception as exc:
658
  return f"Agent initialization failed: {exc}", pd.DataFrame()
659
 
660
+ # Load metadata once before processing questions; only needed if tasks have files.
 
 
 
 
661
  try:
662
  response = requests.get(questions_url, timeout=HTTP_TIMEOUT)
663
  response.raise_for_status()
664
+ questions = response.json()
665
+ if not isinstance(questions, list) or not questions:
666
+ return "Scoring API returned an empty or invalid question list.", pd.DataFrame()
667
  except Exception as exc:
668
+ return f"Could not fetch questions: {exc}", pd.DataFrame()
669
 
670
+ if any(item.get("file_name") for item in questions):
671
+ progress(0.02, desc="Preparing GAIA attachment metadata")
672
+ agent.attachments.initialize()
673
 
674
+ results = []
675
+ answers = []
676
+ total = len(questions)
677
 
678
+ for index, task in enumerate(questions, start=1):
679
+ task_id = task.get("task_id")
680
+ question = task.get("question")
681
+ if not task_id or question is None:
682
  continue
683
 
684
+ progress(
685
+ (index - 1) / max(total, 1),
686
+ desc=f"Solving question {index}/{total}",
687
+ )
688
 
689
  try:
690
+ answer = agent(task)
691
+ answer_for_submission = answer or "Unable to determine"
692
+ display_answer = answer_for_submission
 
 
 
 
 
 
 
693
  except Exception as exc:
694
  print(f"Task {task_id} failed: {exc}")
695
+ answer_for_submission = "Unable to determine"
696
  display_answer = f"AGENT ERROR: {exc}"
 
 
 
 
 
697
 
698
+ answers.append({
699
+ "task_id": task_id,
700
+ "submitted_answer": answer_for_submission,
701
+ })
702
+ results.append({
703
  "Task ID": task_id,
704
+ "Question": question,
705
+ "Attachment": task.get("file_name", ""),
706
  "Submitted Answer": display_answer,
707
  })
708
 
709
+ results_df = pd.DataFrame(results)
710
+ if not answers:
711
+ return "No answers were generated.", results_df
712
 
713
+ space_id = os.getenv("SPACE_ID", "")
714
+ agent_code = (
715
+ f"https://huggingface.co/spaces/{space_id}/tree/main"
716
+ if space_id else ""
717
+ )
718
+ payload = {
719
  "username": username,
720
  "agent_code": agent_code,
721
+ "answers": answers,
722
  }
723
 
724
+ progress(0.96, desc="Submitting answers for scoring")
 
725
  try:
726
  response = requests.post(
727
  submit_url,
728
+ json=payload,
729
+ timeout=150,
730
  )
731
  response.raise_for_status()
732
  result = response.json()
733
+ status = (
 
734
  "Submission successful!\n"
735
  f"User: {result.get('username', username)}\n"
736
  f"Score: {result.get('score', 'N/A')}% "
737
  f"({result.get('correct_count', '?')}/"
738
+ f"{result.get('total_attempted', len(answers))} correct)\n"
739
+ f"Message: {result.get('message', 'Score calculated.')}"
740
  )
741
  progress(1.0, desc="Finished")
742
+ return status, results_df
 
743
  except requests.HTTPError as exc:
744
+ return (
745
+ f"Submission failed: HTTP {exc.response.status_code}: "
746
+ f"{exc.response.text[:1000]}",
747
+ results_df,
748
+ )
 
 
 
 
 
 
749
  except Exception as exc:
750
+ return f"Submission failed: {exc}", results_df
751
 
752
 
753
  # ---------------------------------------------------------------------
754
+ # Gradio UI
755
  # ---------------------------------------------------------------------
756
  with gr.Blocks() as demo:
757
  gr.Markdown("# GAIA Final Assignment — GPT-4o Agent")
758
  gr.Markdown(
759
  """
760
+ This agent uses OpenAI GPT-4o, web research, attachment processing, and
761
+ best-effort YouTube subtitle/frame retrieval.
762
+
763
+ **Required Space secrets:**
764
+ - `OPENAI_API_KEY`: OpenAI API key with available API billing.
765
+ - `HF_TOKEN`: Hugging Face read token. Your account must have accepted the
766
+ access conditions for the gated `gaia-benchmark/GAIA` dataset.
767
 
768
+ The HF token is used only to retrieve GAIA attachments when the course
769
+ scoring Space's `/files/{task_id}` endpoint is unavailable. The course's
770
+ official `/questions` and `/submit` endpoints remain in use.
 
 
771
 
772
+ This agent uses remote OpenAI inference. The ZeroGPU startup probe is not called.
 
773
  """
774
  )
 
775
  gr.LoginButton()
776
  run_button = gr.Button(
777
  "Run Evaluation & Submit All Answers",
 
786
  label="Questions and Agent Answers",
787
  wrap=True,
788
  )
 
789
  run_button.click(
790
  fn=run_and_submit_all,
791
  outputs=[status_output, results_table],
 
793
 
794
 
795
  if __name__ == "__main__":
796
+ print("Starting GAIA GPT-4o evaluation Space.")
797
  print(f"Model: {OPENAI_MODEL}")
798
  print(f"Space ID: {os.getenv('SPACE_ID', 'local')}")
799
+ demo.launch(debug=True, share=False)