vskchandra commited on
Commit
e16d44c
·
verified ·
1 Parent(s): 3513fcd

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +865 -1209
app.py CHANGED
@@ -4,13 +4,16 @@ import re
4
  import subprocess
5
  import sys
6
  import time
 
7
  from pathlib import Path
8
  from urllib.parse import parse_qs, urlparse
9
 
 
10
  import gradio as gr
11
  import pandas as pd
12
  import requests
13
  import spaces
 
14
  from ddgs import DDGS
15
  from google import genai
16
  from google.genai import types
@@ -26,16 +29,20 @@ def _zerogpu_probe():
26
 
27
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
28
  GEMINI_MODEL = "gemini-3.5-flash-lite"
 
29
 
30
  GAIA_ATTACHMENT_FALLBACK_BASE = (
31
  "https://huggingface.co/spaces/bstraehle/gaia/"
32
  "resolve/809adb7a8f1747286eac400b8675dffdd973fbb9/data"
33
  )
34
 
35
- ATTACHMENT_DIR = Path("/tmp/gaia_attachments_v6")
36
  ATTACHMENT_DIR.mkdir(parents=True, exist_ok=True)
37
 
38
- CACHE_FILE = Path("/tmp/gaia_answer_cache_v6.json")
 
 
 
39
 
40
  MEDIA_EXTENSIONS = {
41
  ".png",
@@ -49,6 +56,16 @@ MEDIA_EXTENSIONS = {
49
  ".ogg",
50
  }
51
 
 
 
 
 
 
 
 
 
 
 
52
 
53
  class AnswerResponse(BaseModel):
54
  answer: str
@@ -58,18 +75,23 @@ class SearchPlan(BaseModel):
58
  queries: list[str]
59
 
60
 
 
 
 
 
 
 
 
 
 
61
  def load_cache() -> dict:
62
  if not CACHE_FILE.exists():
63
  return {}
64
 
65
  try:
66
- data = json.loads(
67
- CACHE_FILE.read_text(encoding="utf-8")
68
- )
69
-
70
  if isinstance(data, dict):
71
  return data
72
-
73
  except Exception as error:
74
  print(f"Could not load cache: {error}")
75
 
@@ -78,11 +100,7 @@ def load_cache() -> dict:
78
 
79
  def save_cache(cache: dict):
80
  CACHE_FILE.write_text(
81
- json.dumps(
82
- cache,
83
- ensure_ascii=False,
84
- indent=2,
85
- ),
86
  encoding="utf-8",
87
  )
88
 
@@ -110,72 +128,55 @@ def clean_answer(answer: str) -> str:
110
 
111
  if answer.startswith("```") and answer.endswith("```"):
112
  answer = answer[3:-3].strip()
113
-
114
  first_line, separator, remainder = answer.partition("\n")
115
-
116
- if (
117
- separator
118
- and first_line.strip().lower()
119
- in {
120
- "text",
121
- "plaintext",
122
- "python",
123
- "json",
124
- }
125
- ):
126
  answer = remainder.strip()
127
 
128
  return answer.strip("`").strip()
129
 
130
 
131
  def normalized_answer(answer: str) -> str:
132
- return re.sub(
133
- r"\s+",
134
- " ",
135
- clean_answer(answer),
136
- ).strip().casefold()
137
 
138
 
139
  def extract_youtube_id(text: str):
140
- urls = re.findall(
141
- r"https?://[^\s)>]+",
142
- text,
143
- )
144
 
145
  for url in urls:
146
  url = url.rstrip(".,;!?")
147
-
148
  try:
149
  parsed = urlparse(url)
150
-
151
  if "youtube.com" in parsed.netloc:
152
- video_id = parse_qs(
153
- parsed.query
154
- ).get("v")
155
-
156
  if video_id:
157
  return video_id[0]
158
-
159
  if "youtu.be" in parsed.netloc:
160
  value = parsed.path.strip("/")
161
-
162
  if value:
163
  return value
164
-
165
  except Exception:
166
  continue
167
 
168
  return None
169
 
170
 
 
 
 
 
 
 
 
171
  class BasicAgent:
172
  def __init__(self):
173
  api_key = os.getenv("GEMINI_API_KEY")
174
-
175
  if not api_key:
176
- raise RuntimeError(
177
- "GEMINI_API_KEY secret is not configured."
178
- )
179
 
180
  self.client = genai.Client(
181
  api_key=api_key,
@@ -186,143 +187,71 @@ class BasicAgent:
186
  exp_base=2.0,
187
  max_delay=30.0,
188
  jitter=1.0,
189
- http_status_codes=[
190
- 408,
191
- 500,
192
- 502,
193
- 503,
194
- 504,
195
- ],
196
  )
197
  ),
198
  )
199
 
200
- print(
201
- f"BasicAgent initialized with {GEMINI_MODEL}."
 
 
 
 
 
 
202
  )
203
 
204
- def download_attachment(
205
- self,
206
- task_id: str,
207
- file_name: str,
208
- ) -> Path:
209
  safe_name = os.path.basename(file_name)
210
-
211
  if not safe_name:
212
- raise RuntimeError(
213
- "Attachment file name is empty."
214
- )
215
-
216
- destination = (
217
- ATTACHMENT_DIR
218
- / f"{task_id}_{safe_name}"
219
- )
220
-
221
- if (
222
- destination.exists()
223
- and destination.stat().st_size > 0
224
- ):
225
- print(
226
- f"Using cached attachment: {safe_name}"
227
- )
228
 
 
 
 
229
  return destination
230
 
231
- official_url = (
232
- f"{DEFAULT_API_URL}/files/{task_id}"
233
- )
234
-
235
  official_error = None
236
 
237
  try:
238
- print(
239
- "Trying official attachment endpoint: "
240
- f"{official_url}"
241
- )
242
-
243
- response = requests.get(
244
- official_url,
245
- timeout=30,
246
- )
247
-
248
  response.raise_for_status()
249
-
250
  if not response.content:
251
- raise RuntimeError(
252
- "Official endpoint returned an empty file."
253
- )
254
-
255
- destination.write_bytes(
256
- response.content
257
- )
258
-
259
- print(
260
- "Downloaded attachment from official endpoint: "
261
- f"{safe_name}"
262
- )
263
-
264
  return destination
265
-
266
  except Exception as error:
267
  official_error = error
268
-
269
  print(
270
  "Official attachment download failed: "
271
  f"{type(error).__name__}: {error}"
272
  )
273
 
274
  fallback_url = (
275
- f"{GAIA_ATTACHMENT_FALLBACK_BASE}/"
276
- f"{safe_name}?download=true"
277
- )
278
-
279
- print(
280
- f"Trying attachment fallback: {fallback_url}"
281
  )
 
282
 
283
  try:
284
- response = requests.get(
285
- fallback_url,
286
- timeout=60,
287
- )
288
-
289
  response.raise_for_status()
290
-
291
  if not response.content:
292
- raise RuntimeError(
293
- "Fallback returned an empty file."
294
- )
295
-
296
- content_type = (
297
- response.headers
298
- .get("content-type", "")
299
- .lower()
300
- )
301
-
302
- suffix = Path(
303
- safe_name
304
- ).suffix.lower()
305
 
306
- if (
307
- "text/html" in content_type
308
- and suffix in MEDIA_EXTENSIONS
309
- ):
310
  raise RuntimeError(
311
- "Fallback returned HTML instead "
312
- "of the requested attachment."
313
  )
314
 
315
- destination.write_bytes(
316
- response.content
317
- )
318
-
319
- print(
320
- "Downloaded attachment from fallback: "
321
- f"{safe_name}"
322
- )
323
-
324
  return destination
325
-
326
  except Exception as fallback_error:
327
  raise RuntimeError(
328
  "Unable to download attachment. "
@@ -330,40 +259,21 @@ class BasicAgent:
330
  f"Fallback error: {fallback_error}"
331
  )
332
 
333
- def solve_reversed_text(
334
- self,
335
- question: str,
336
- ):
337
  reversed_question = question[::-1]
338
-
339
  lowered = reversed_question.lower()
340
 
341
- if (
342
- "opposite of the word" not in lowered
343
- and "opposite of" not in lowered
344
- ):
345
  return None
346
 
347
- patterns = [
 
348
  r'opposite of the word\s+["\']([^"\']+)["\']',
349
  r'opposite of\s+["\']([^"\']+)["\']',
350
- ]
351
-
352
- target = None
353
-
354
- for pattern in patterns:
355
- match = re.search(
356
- pattern,
357
- reversed_question,
358
- flags=re.IGNORECASE,
359
- )
360
-
361
  if match:
362
- target = (
363
- match.group(1)
364
- .strip()
365
- .lower()
366
- )
367
  break
368
 
369
  if not target:
@@ -385,24 +295,13 @@ class BasicAgent:
385
  }
386
 
387
  answer = opposites.get(target)
388
-
389
  if answer:
390
- print(
391
- f"Local reversed-text solver: {answer}"
392
- )
393
-
394
  return answer
395
 
396
- def solve_operation_table(
397
- self,
398
- question: str,
399
- ):
400
  lowered = question.lower()
401
-
402
- if (
403
- "commutative" not in lowered
404
- or "|" not in question
405
- ):
406
  return None
407
 
408
  set_match = re.search(
@@ -410,7 +309,6 @@ class BasicAgent:
410
  question,
411
  flags=re.IGNORECASE,
412
  )
413
-
414
  if not set_match:
415
  return None
416
 
@@ -419,167 +317,83 @@ class BasicAgent:
419
  for value in set_match.group(1).split(",")
420
  if value.strip()
421
  ]
422
-
423
  if len(elements) < 2:
424
  return None
425
 
426
- raw_tokens = re.findall(
427
- r"\|([^|]*)",
428
- question,
429
- )
430
-
431
- tokens = [
432
- token.strip()
433
- for token in raw_tokens
434
- ]
435
-
436
  rows = {}
437
  n = len(elements)
438
-
439
  index = 0
440
 
441
  while index < len(tokens):
442
- if (
443
- tokens[index] in elements
444
- and index + n < len(tokens)
445
- ):
446
- candidate = tokens[
447
- index + 1:
448
- index + 1 + n
449
- ]
450
-
451
- if (
452
- len(candidate) == n
453
- and all(
454
- value in elements
455
- for value in candidate
456
- )
457
- ):
458
- rows[
459
- tokens[index]
460
- ] = dict(
461
- zip(
462
- elements,
463
- candidate,
464
- )
465
- )
466
-
467
  index += n + 1
468
  continue
469
-
470
  index += 1
471
 
472
  if len(rows) != n:
473
  return None
474
 
475
  involved = set()
476
-
477
  for left in elements:
478
  for right in elements:
479
- if (
480
- left in rows
481
- and right in rows
482
- and rows[left].get(right)
483
- != rows[right].get(left)
484
- ):
485
  involved.add(left)
486
  involved.add(right)
487
 
488
  if not involved:
489
  return None
490
 
491
- answer = ", ".join(
492
- sorted(involved)
493
- )
494
-
495
- print(
496
- f"Local operation-table solver: {answer}"
497
- )
498
-
499
  return answer
500
 
501
- def solve_python_file(
502
- self,
503
- path: Path,
504
- question: str,
505
- ):
506
- if (
507
- path.suffix.lower() != ".py"
508
- or "numeric output"
509
- not in question.lower()
510
- ):
511
  return None
512
 
513
  try:
514
  completed = subprocess.run(
515
- [
516
- sys.executable,
517
- "-I",
518
- str(path),
519
- ],
520
  cwd=str(path.parent),
521
  capture_output=True,
522
  text=True,
523
  timeout=15,
524
  )
525
-
526
  except Exception as error:
527
- print(
528
- f"Local Python execution failed: {error}"
529
- )
530
  return None
531
 
532
  stdout = completed.stdout.strip()
533
-
534
  if not stdout:
535
- print(
536
- "Python script produced no stdout."
537
- )
538
  return None
539
 
540
- lines = [
541
- line.strip()
542
- for line in stdout.splitlines()
543
- if line.strip()
544
- ]
545
-
546
  if not lines:
547
  return None
548
 
549
- last_line = lines[-1]
550
-
551
  numeric_matches = re.findall(
552
  r"-?(?:\d+(?:\.\d+)?|\.\d+)",
553
- last_line,
554
  )
555
-
556
  if not numeric_matches:
557
  return None
558
 
559
  answer = numeric_matches[-1]
560
-
561
- print(
562
- f"Local Python solver: {answer}"
563
- )
564
-
565
  return answer
566
 
567
- def solve_excel_food_sales(
568
- self,
569
- path: Path,
570
- question: str,
571
- ):
572
  lowered = question.lower()
573
-
574
  if (
575
- path.suffix.lower()
576
- not in {".xlsx", ".xlsm"}
577
  or "sales" not in lowered
578
  or "food" not in lowered
579
- or (
580
- "drink" not in lowered
581
- and "beverage" not in lowered
582
- )
583
  ):
584
  return None
585
 
@@ -621,248 +435,624 @@ class BasicAgent:
621
 
622
  try:
623
  workbook = pd.ExcelFile(path)
624
-
625
  grand_total = 0.0
626
  found_food_columns = 0
627
  found_drink_columns = 0
628
 
629
  for sheet in workbook.sheet_names:
630
- frame = pd.read_excel(
631
- path,
632
- sheet_name=sheet,
633
- )
634
-
635
  if frame.empty:
636
  continue
637
 
638
- numeric_columns = []
639
-
640
  for column in frame.columns:
641
- converted = pd.to_numeric(
642
- frame[column],
643
- errors="coerce",
644
- )
645
-
646
- if converted.notna().any():
647
- numeric_columns.append(
648
- (
649
- column,
650
- converted,
651
- )
652
- )
653
-
654
- for column, values in numeric_columns:
655
- column_name = str(
656
- column
657
- ).strip().lower()
658
-
659
- if any(
660
- word in column_name
661
- for word in metadata_words
662
- ):
663
  continue
664
 
665
- if any(
666
- word in column_name
667
- for word in drink_words
668
- ):
669
  found_drink_columns += 1
670
  continue
671
 
672
- grand_total += float(
673
- values.fillna(0).sum()
674
- )
675
-
676
  found_food_columns += 1
677
 
678
- if (
679
- found_food_columns == 0
680
- or found_drink_columns == 0
681
- ):
682
  return None
683
 
684
  answer = f"{grand_total:.2f}"
685
-
686
  print(
687
- "Local Excel solver:"
688
- f" food columns={found_food_columns},"
689
- f" drink columns={found_drink_columns},"
690
- f" answer={answer}"
691
  )
692
-
693
  return answer
694
-
695
  except Exception as error:
696
- print(
697
- f"Local Excel calculation failed: {error}"
698
- )
699
 
 
 
 
700
  return None
701
 
702
- def workbook_to_text(
703
- self,
704
- path: Path,
705
- ) -> str:
706
- workbook = load_workbook(
707
- path,
708
- data_only=True,
709
- read_only=True,
710
- )
711
 
712
- output = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
713
 
714
  try:
715
- for sheet_name in workbook.sheetnames:
716
- sheet = workbook[
717
- sheet_name
718
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
719
 
720
- output.append(
721
- f"=== SHEET: {sheet_name} ==="
 
 
 
 
 
 
 
 
 
 
 
 
722
  )
 
 
 
 
723
 
724
- max_rows = min(
725
- sheet.max_row or 0,
726
- 1000,
 
 
 
 
 
 
 
 
 
 
727
  )
 
 
 
728
 
729
- max_columns = min(
730
- sheet.max_column or 0,
731
- 80,
 
 
 
 
 
 
 
732
  )
 
 
733
 
734
- for row_number in range(
735
- 1,
736
- max_rows + 1,
737
- ):
738
- values = []
739
 
740
- for column_number in range(
741
- 1,
742
- max_columns + 1,
743
- ):
744
- value = sheet.cell(
745
- row=row_number,
746
- column=column_number,
747
- ).value
748
 
749
- if value is None:
750
- values.append("")
751
- else:
752
- values.append(
753
- str(value)
754
- )
755
 
756
- if any(values):
757
- output.append(
758
- "\t".join(values)
759
- )
760
 
761
- finally:
762
- workbook.close()
763
 
764
- text = "\n".join(output)
 
 
765
 
766
- return text[:70000]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
767
 
768
- def fetch_youtube_transcript(
769
- self,
770
- question: str,
771
- ):
772
- video_id = extract_youtube_id(
773
- question
774
  )
775
 
776
- if not video_id:
777
- return ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
778
 
779
  print(
780
- f"Trying YouTube transcript: {video_id}"
 
781
  )
782
 
 
 
 
 
 
 
783
  try:
784
- api = YouTubeTranscriptApi()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
785
 
786
- transcript = api.fetch(
787
- video_id,
788
- languages=["en"],
 
 
 
 
 
 
 
 
 
 
 
789
  )
790
 
791
- text = " ".join(
792
- snippet.text
793
- for snippet in transcript
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
794
  )
795
 
796
- if text.strip():
797
- print(
798
- f"YouTube transcript obtained: "
799
- f"{len(text)} chars"
800
- )
801
 
802
- return text[:60000]
 
 
803
 
804
- except Exception as first_error:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
805
  print(
806
- "Direct English transcript failed: "
807
- f"{first_error}"
808
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
809
 
810
  try:
811
- api = YouTubeTranscriptApi()
 
 
 
 
812
 
813
- transcript_list = api.list(
814
- video_id
815
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
816
 
 
 
 
817
  for transcript in transcript_list:
818
  try:
819
  if transcript.language_code == "en":
820
  fetched = transcript.fetch()
821
-
822
  elif transcript.is_translatable:
823
- fetched = (
824
- transcript
825
- .translate("en")
826
- .fetch()
827
- )
828
-
829
  else:
830
  continue
831
 
832
- text = " ".join(
833
- snippet.text
834
- for snippet in fetched
835
- )
836
-
837
  if text.strip():
838
  print(
839
- "Fallback YouTube transcript "
840
- f"obtained: {len(text)} chars"
841
  )
842
-
843
  return text[:60000]
844
-
845
  except Exception:
846
  continue
847
-
848
  except Exception as error:
849
- print(
850
- f"YouTube transcript unavailable: {error}"
851
- )
852
 
853
  return ""
854
 
855
- def plan_search_queries(
856
- self,
857
- question: str,
858
- ):
859
  prompt = f"""
860
- Create 3 concise web search queries that would help solve
861
- this factual research question.
862
-
863
- Do not answer the question.
864
- Do not search for the benchmark or task ID.
865
- Search for the underlying real-world information.
866
 
867
  QUESTION:
868
  {question}
@@ -880,244 +1070,116 @@ QUESTION:
880
  )
881
 
882
  if response.parsed:
883
- if isinstance(
884
- response.parsed,
885
- SearchPlan,
886
- ):
887
  plan = response.parsed
888
  else:
889
- plan = SearchPlan.model_validate(
890
- response.parsed
891
- )
892
-
893
  else:
894
- plan = SearchPlan.model_validate_json(
895
- response.text
896
- )
897
-
898
- queries = [
899
- query.strip()
900
- for query in plan.queries
901
- if query.strip()
902
- ]
903
 
 
904
  if queries:
905
- print(
906
- "Search queries: "
907
- + " | ".join(queries[:3])
908
- )
909
-
910
  return queries[:3]
911
-
912
  except Exception as error:
913
- print(
914
- f"Search-query planning failed: {error}"
915
- )
916
-
917
- simplified = re.sub(
918
- r"https?://\S+",
919
- " ",
920
- question,
921
- )
922
-
923
- simplified = re.sub(
924
- r"\s+",
925
- " ",
926
- simplified,
927
- ).strip()
928
-
929
- return [
930
- simplified[:250]
931
- ]
932
 
933
- def web_research(
934
- self,
935
- question: str,
936
- ) -> str:
937
- queries = self.plan_search_queries(
938
- question
939
- )
940
 
 
 
941
  results_output = []
942
  urls = []
943
  seen_urls = set()
944
 
945
- for query_number, query in enumerate(
946
- queries,
947
- start=1,
948
- ):
949
  try:
950
- results = DDGS(
951
- timeout=15,
952
- ).text(
953
  query,
954
  region="us-en",
955
  safesearch="moderate",
956
  max_results=8,
957
  )
958
-
959
  except Exception as error:
960
- print(
961
- f"Search failed for '{query}': {error}"
962
- )
963
- continue
964
-
965
- if not results:
966
  continue
967
 
968
- for index, result in enumerate(
969
- results,
970
- start=1,
971
- ):
972
- title = (
973
- result.get("title")
974
- or ""
975
- )
976
-
977
- url = (
978
- result.get("href")
979
- or result.get("url")
980
- or ""
981
- )
982
-
983
- snippet = (
984
- result.get("body")
985
- or result.get("description")
986
- or ""
987
- )
988
 
989
  results_output.append(
990
- f"\nSEARCH QUERY {query_number}, "
991
- f"RESULT {index}\n"
992
  f"Title: {title}\n"
993
  f"URL: {url}\n"
994
  f"Snippet: {snippet}\n"
995
  )
996
 
997
- if (
998
- url
999
- and url not in seen_urls
1000
- and len(urls) < 6
1001
- ):
1002
  urls.append(url)
1003
  seen_urls.add(url)
1004
 
1005
- for index, url in enumerate(
1006
- urls,
1007
- start=1,
1008
- ):
1009
  try:
1010
- page = DDGS(
1011
- timeout=15,
1012
- ).extract(
1013
- url,
1014
- fmt="text_plain",
1015
- )
1016
-
1017
- if isinstance(
1018
- page,
1019
- dict,
1020
- ):
1021
- page_text = str(
1022
- page.get(
1023
- "content",
1024
- "",
1025
- )
1026
- )
1027
-
1028
  else:
1029
- page_text = str(
1030
- page or ""
1031
- )
1032
 
1033
  if not page_text.strip():
1034
  continue
1035
 
1036
- page_text = page_text[:10000]
1037
-
1038
  results_output.append(
1039
- f"\nWEB PAGE {index}\n"
1040
- f"URL: {url}\n"
1041
- f"{page_text}\n"
1042
  )
1043
-
1044
  except Exception as error:
1045
- print(
1046
- f"Could not read {url}: {error}"
1047
- )
1048
-
1049
- research = "\n".join(
1050
- results_output
1051
- )
1052
 
 
1053
  if not research.strip():
1054
- return (
1055
- "No useful web research could be retrieved."
1056
- )
1057
 
1058
  return research[:50000]
1059
 
1060
- def system_instruction(
1061
- self,
1062
- ):
1063
  return """
1064
  You are solving one GAIA Level 1 benchmark task.
1065
 
1066
  Accuracy is more important than speed.
1067
-
1068
  Use only the question and evidence supplied in this request.
 
 
1069
  Do not assume the first search result is correct.
1070
  Cross-check names, dates, numbers and relationships.
1071
 
1072
  The evaluator uses exact-match scoring.
1073
-
1074
  Return ONLY the requested final answer.
1075
-
1076
- Do not include:
1077
- - reasoning
1078
- - explanations
1079
- - citations
1080
- - Markdown
1081
- - "Final answer"
1082
- - "The answer is"
1083
-
1084
- Follow the requested format exactly.
1085
-
1086
- Important formatting rules:
1087
- - Preserve canonical capitalization of names and words.
1088
- - Do not arbitrarily lowercase answers.
1089
- - If the question asks for a city without abbreviations,
1090
- spell the city name out fully.
1091
- - If the question requests first name only, return first name only.
1092
- - If the question requests surname only, return surname only.
1093
- - If the question requests an IOC code, return only that code.
1094
- - If the question requests a comma-separated list, use exactly
1095
- that format and preserve the requested ordering.
1096
- - If it asks for alphabetical order, alphabetize exactly.
1097
- - For numeric answers, do not add units or symbols unless the
1098
- requested output explicitly requires them.
1099
- - If USD is requested with two decimal places but no explicit
1100
- request for a currency symbol, return the number with exactly
1101
- two decimal places.
1102
- - Carefully distinguish similar people, titles and dates.
1103
  """
1104
 
1105
- def generate_answer(
1106
- self,
1107
- question: str,
1108
- evidence: str,
1109
- media=None,
1110
- independent=False,
1111
- ):
1112
- if independent:
1113
- instruction = """
1114
- Solve the question independently from the evidence.
1115
- Do not assume any previous candidate answer is correct.
1116
- """
1117
- else:
1118
- instruction = """
1119
- Solve the question carefully from the evidence.
1120
- """
1121
 
1122
  prompt = f"""
1123
  {instruction}
@@ -1131,10 +1193,7 @@ EVIDENCE:
1131
  Return only the exact requested answer.
1132
  """
1133
 
1134
- contents = [
1135
- prompt
1136
- ]
1137
-
1138
  if media is not None:
1139
  contents.append(media)
1140
 
@@ -1142,60 +1201,30 @@ Return only the exact requested answer.
1142
  model=GEMINI_MODEL,
1143
  contents=contents,
1144
  config=types.GenerateContentConfig(
1145
- system_instruction=
1146
- self.system_instruction(),
1147
  max_output_tokens=2048,
1148
- thinking_config=
1149
- types.ThinkingConfig(
1150
- thinking_level="medium"
1151
- ),
1152
- response_mime_type=
1153
- "application/json",
1154
- response_schema=
1155
- AnswerResponse,
1156
  ),
1157
  )
1158
 
1159
  if response.parsed:
1160
- if isinstance(
1161
- response.parsed,
1162
- AnswerResponse,
1163
- ):
1164
  parsed = response.parsed
1165
  else:
1166
- parsed = AnswerResponse.model_validate(
1167
- response.parsed
1168
- )
1169
-
1170
  else:
1171
  if not response.text:
1172
- raise RuntimeError(
1173
- "Gemini returned an empty response."
1174
- )
1175
-
1176
- parsed = AnswerResponse.model_validate_json(
1177
- response.text
1178
- )
1179
-
1180
- answer = clean_answer(
1181
- parsed.answer
1182
- )
1183
 
 
1184
  if not answer:
1185
- raise RuntimeError(
1186
- "Gemini returned an empty answer."
1187
- )
1188
-
1189
  return answer
1190
 
1191
- def adjudicate(
1192
- self,
1193
- question: str,
1194
- evidence: str,
1195
- first_answer: str,
1196
- second_answer: str,
1197
- media=None,
1198
- ):
1199
  prompt = f"""
1200
  Two independent attempts produced different answers.
1201
 
@@ -1211,18 +1240,12 @@ CANDIDATE A:
1211
  CANDIDATE B:
1212
  {second_answer}
1213
 
1214
- Re-solve the problem from the evidence.
1215
- Do not choose based on majority or wording.
1216
- Determine which answer is factually correct, or produce a
1217
- different answer if both candidates are wrong.
1218
-
1219
- Return only the exact requested answer.
1220
  """
1221
 
1222
- contents = [
1223
- prompt
1224
- ]
1225
-
1226
  if media is not None:
1227
  contents.append(media)
1228
 
@@ -1230,56 +1253,32 @@ Return only the exact requested answer.
1230
  model=GEMINI_MODEL,
1231
  contents=contents,
1232
  config=types.GenerateContentConfig(
1233
- system_instruction=
1234
- self.system_instruction(),
1235
  max_output_tokens=2048,
1236
- thinking_config=
1237
- types.ThinkingConfig(
1238
- thinking_level="medium"
1239
- ),
1240
- response_mime_type=
1241
- "application/json",
1242
- response_schema=
1243
- AnswerResponse,
1244
  ),
1245
  )
1246
 
1247
  if response.parsed:
1248
- if isinstance(
1249
- response.parsed,
1250
- AnswerResponse,
1251
- ):
1252
  parsed = response.parsed
1253
  else:
1254
- parsed = AnswerResponse.model_validate(
1255
- response.parsed
1256
- )
1257
-
1258
  else:
1259
- parsed = AnswerResponse.model_validate_json(
1260
- response.text
1261
- )
1262
 
1263
- return clean_answer(
1264
- parsed.answer
1265
- )
1266
 
1267
- def solve_with_consensus(
1268
- self,
1269
- question: str,
1270
- evidence: str,
1271
- media=None,
1272
- ):
1273
  first = self.generate_answer(
1274
  question,
1275
  evidence,
1276
  media,
1277
  independent=False,
1278
  )
1279
-
1280
- print(
1281
- f"First model answer: {first}"
1282
- )
1283
 
1284
  time.sleep(1.0)
1285
 
@@ -1289,26 +1288,13 @@ Return only the exact requested answer.
1289
  media,
1290
  independent=True,
1291
  )
 
1292
 
1293
- print(
1294
- f"Independent model answer: {second}"
1295
- )
1296
-
1297
- if (
1298
- normalized_answer(first)
1299
- == normalized_answer(second)
1300
- ):
1301
- print(
1302
- "Both model attempts agree."
1303
- )
1304
-
1305
  return first
1306
 
1307
- print(
1308
- "Model attempts disagree. "
1309
- "Running adjudication."
1310
- )
1311
-
1312
  time.sleep(1.0)
1313
 
1314
  final = self.adjudicate(
@@ -1318,580 +1304,280 @@ Return only the exact requested answer.
1318
  second,
1319
  media,
1320
  )
1321
-
1322
- print(
1323
- f"Adjudicated answer: {final}"
1324
- )
1325
-
1326
  return final
1327
 
1328
- def solve_task(
1329
- self,
1330
- task: dict,
1331
- ):
1332
  task_id = task["task_id"]
1333
  question = task["question"]
1334
  file_name = task["file_name"]
1335
 
1336
  print("")
1337
- print(
1338
- f"Solving task {task_id}"
1339
- )
1340
-
1341
- reversed_answer = (
1342
- self.solve_reversed_text(
1343
- question
1344
- )
1345
- )
1346
-
1347
- if reversed_answer is not None:
1348
- return reversed_answer
1349
-
1350
- table_answer = (
1351
- self.solve_operation_table(
1352
- question
1353
- )
1354
- )
1355
-
1356
- if table_answer is not None:
1357
- return table_answer
1358
 
1359
  path = None
1360
-
1361
  if file_name:
1362
- path = self.download_attachment(
1363
- task_id,
1364
- file_name,
1365
- )
1366
-
1367
- if (
1368
- path is not None
1369
- and path.suffix.lower() == ".py"
1370
- ):
1371
- python_answer = (
1372
- self.solve_python_file(
1373
- path,
1374
- question,
1375
- )
1376
- )
1377
 
1378
- if python_answer is not None:
1379
- return python_answer
 
 
1380
 
1381
- if (
1382
- path is not None
1383
- and path.suffix.lower()
1384
- in {".xlsx", ".xlsm"}
1385
- ):
1386
- excel_answer = (
1387
- self.solve_excel_food_sales(
1388
- path,
1389
- question,
1390
- )
1391
- )
1392
 
1393
- if excel_answer is not None:
1394
- return excel_answer
1395
-
1396
- youtube_transcript = (
1397
- self.fetch_youtube_transcript(
1398
- question
1399
- )
1400
- )
1401
 
 
1402
  if youtube_transcript:
1403
  evidence = (
1404
- "The following is the transcript "
1405
- "of the YouTube video referenced "
1406
- "in the question:\n\n"
1407
- + youtube_transcript
1408
- )
1409
-
1410
- return self.solve_with_consensus(
1411
- question,
1412
- evidence,
1413
- )
1414
-
1415
- if (
1416
- path is not None
1417
- and path.suffix.lower()
1418
- in MEDIA_EXTENSIONS
1419
- ):
1420
- print(
1421
- f"Uploading media: {file_name}"
1422
- )
1423
-
1424
- media = self.client.files.upload(
1425
- file=str(path)
1426
  )
 
1427
 
1428
- print(
1429
- "Media upload completed."
1430
- )
 
1431
 
1432
  evidence = (
1433
- "The actual media attachment referenced "
1434
- "by the question is included in this request. "
1435
- "Inspect the attachment itself carefully. "
1436
  f"Filename: {file_name}"
1437
  )
 
1438
 
1439
- return self.solve_with_consensus(
1440
- question,
1441
- evidence,
1442
- media,
1443
- )
1444
-
1445
- if (
1446
- path is not None
1447
- and path.suffix.lower()
1448
- in {".xlsx", ".xlsm"}
1449
- ):
1450
- workbook_text = (
1451
- self.workbook_to_text(
1452
- path
1453
- )
1454
- )
1455
-
1456
  evidence = (
1457
- "The following data was extracted from "
1458
- "the attached workbook:\n\n"
1459
  + workbook_text
1460
  )
 
1461
 
1462
- return self.solve_with_consensus(
1463
- question,
1464
- evidence,
1465
- )
1466
-
1467
- if (
1468
- path is not None
1469
- and path.suffix.lower()
1470
- in {
1471
- ".txt",
1472
- ".csv",
1473
- ".json",
1474
- ".md",
1475
- ".py",
1476
- }
1477
- ):
1478
- file_text = path.read_text(
1479
- encoding="utf-8",
1480
- errors="replace",
1481
- )[:70000]
1482
-
1483
  evidence = (
1484
- "The following is the content of the "
1485
- "attached file:\n\n"
1486
  + file_text
1487
  )
 
1488
 
1489
- return self.solve_with_consensus(
1490
- question,
1491
- evidence,
1492
- )
1493
-
1494
- print(
1495
- "Running targeted web research."
1496
- )
1497
-
1498
- research = self.web_research(
1499
- question
1500
- )
1501
-
1502
- print(
1503
- f"Web research collected: "
1504
- f"{len(research)} chars"
1505
- )
1506
-
1507
- return self.solve_with_consensus(
1508
- question,
1509
- research,
1510
- )
1511
 
1512
 
1513
- def build_results_table(
1514
- tasks: list[dict],
1515
- cache: dict,
1516
- ):
1517
  rows = []
1518
 
1519
  for task in tasks:
1520
- task_id = task[
1521
- "task_id"
1522
- ]
1523
-
1524
  rows.append(
1525
  {
1526
- "Task ID":
1527
- task_id,
1528
- "Question":
1529
- task[
1530
- "question"
1531
- ],
1532
- "File":
1533
- task[
1534
- "file_name"
1535
- ],
1536
- "Answer":
1537
- cache.get(
1538
- task_id,
1539
- "",
1540
- ),
1541
- "Status":
1542
- (
1543
- "READY"
1544
- if task_id in cache
1545
- else "PENDING"
1546
- ),
1547
  }
1548
  )
1549
 
1550
- return pd.DataFrame(
1551
- rows
1552
- )
1553
 
1554
 
1555
- def run_and_submit_all(
1556
- profile: gr.OAuthProfile | None,
1557
- ):
1558
  if not profile:
1559
- return (
1560
- "Please Login to Hugging Face.",
1561
- None,
1562
- )
1563
-
1564
- username = str(
1565
- profile.username
1566
- ).strip()
1567
 
1568
- space_id = os.getenv(
1569
- "SPACE_ID"
1570
- )
1571
 
1572
  if not space_id:
1573
- return (
1574
- "SPACE_ID environment variable is missing.",
1575
- None,
1576
- )
1577
-
1578
- questions_url = (
1579
- f"{DEFAULT_API_URL}/questions"
1580
- )
1581
 
1582
- submit_url = (
1583
- f"{DEFAULT_API_URL}/submit"
1584
- )
1585
 
1586
  try:
1587
  response = requests.get(
1588
  questions_url,
 
1589
  timeout=30,
1590
  )
1591
-
1592
  response.raise_for_status()
1593
-
1594
- questions_data = (
1595
- response.json()
1596
- )
1597
-
1598
  except Exception as error:
1599
- return (
1600
- f"Could not fetch questions: {error}",
1601
- None,
1602
- )
1603
 
1604
  if not questions_data:
1605
- return (
1606
- "No evaluation questions were returned.",
1607
- None,
1608
- )
1609
 
1610
  tasks = []
1611
-
1612
  for item in questions_data:
1613
- task_id = item.get(
1614
- "task_id"
1615
- )
1616
-
1617
- question = item.get(
1618
- "question"
1619
- )
1620
-
1621
- file_name = (
1622
- item.get(
1623
- "file_name"
1624
- )
1625
- or ""
1626
- )
1627
 
1628
- if (
1629
- not task_id
1630
- or question is None
1631
- ):
1632
- return (
1633
- "The questions API returned "
1634
- "an invalid task.",
1635
- None,
1636
- )
1637
 
1638
  tasks.append(
1639
  {
1640
- "task_id":
1641
- task_id,
1642
- "question":
1643
- question,
1644
- "file_name":
1645
- file_name,
1646
  }
1647
  )
1648
 
1649
  try:
1650
  agent = BasicAgent()
1651
-
1652
  except Exception as error:
1653
- return (
1654
- f"Could not initialize agent: {error}",
1655
- None,
1656
- )
1657
 
1658
  cache = load_cache()
1659
-
1660
- valid_ids = {
1661
- task["task_id"]
1662
- for task in tasks
1663
- }
1664
-
1665
  cache = {
1666
  task_id: answer
1667
- for task_id, answer
1668
- in cache.items()
1669
  if task_id in valid_ids
1670
  }
1671
-
1672
  save_cache(cache)
1673
 
1674
- print(
1675
- f"Total tasks: {len(tasks)}"
1676
- )
1677
-
1678
- print(
1679
- f"Already solved: {len(cache)}"
1680
- )
1681
-
1682
- print(
1683
- f"Remaining: {len(tasks) - len(cache)}"
1684
- )
1685
 
1686
  try:
1687
- for index, task in enumerate(
1688
- tasks,
1689
- start=1,
1690
- ):
1691
- task_id = task[
1692
- "task_id"
1693
- ]
1694
 
1695
  if task_id in cache:
1696
  print(
1697
- f"Skipping cached task "
1698
- f"{index}/{len(tasks)}: "
1699
- f"{task_id}"
1700
  )
1701
  continue
1702
 
1703
  print("")
1704
- print(
1705
- f"Processing task "
1706
- f"{index}/{len(tasks)}"
1707
- )
1708
-
1709
- answer = agent.solve_task(
1710
- task
1711
- )
1712
-
1713
- answer = clean_answer(
1714
- answer
1715
- )
1716
 
1717
  if not answer:
1718
- raise RuntimeError(
1719
- f"Task {task_id} "
1720
- "produced an empty answer."
1721
- )
1722
-
1723
- cache[
1724
- task_id
1725
- ] = answer
1726
-
1727
- save_cache(
1728
- cache
1729
- )
1730
-
1731
- print(
1732
- f"Saved answer for "
1733
- f"{task_id}: {answer}"
1734
- )
1735
 
 
 
 
1736
  time.sleep(1.0)
1737
 
1738
  except Exception as error:
1739
  print(
1740
  "Evaluation paused: "
1741
- f"{type(error).__name__}: "
1742
- f"{error}"
1743
  )
1744
-
1745
  return (
1746
- (
1747
- "Evaluation paused. "
1748
- "Nothing was submitted.\n\n"
1749
- f"Saved answers: "
1750
- f"{len(cache)}/{len(tasks)}\n"
1751
- f"Remaining: "
1752
- f"{len(tasks) - len(cache)}\n\n"
1753
- f"Error: "
1754
- f"{type(error).__name__}: "
1755
- f"{error}\n\n"
1756
- "Click the same button again "
1757
- "to resume from the saved answers."
1758
- ),
1759
- build_results_table(
1760
- tasks,
1761
- cache,
1762
- ),
1763
  )
1764
 
1765
  missing = [
1766
  task["task_id"]
1767
  for task in tasks
1768
- if task["task_id"]
1769
- not in cache
1770
  ]
1771
 
1772
  if missing:
1773
  return (
1774
- (
1775
- "Evaluation is incomplete. "
1776
- "Nothing was submitted.\n\n"
1777
- f"Saved answers: "
1778
- f"{len(cache)}/{len(tasks)}"
1779
- ),
1780
- build_results_table(
1781
- tasks,
1782
- cache,
1783
- ),
1784
  )
1785
 
1786
  answers_payload = [
1787
  {
1788
- "task_id":
1789
- task["task_id"],
1790
- "submitted_answer":
1791
- cache[
1792
- task["task_id"]
1793
- ],
1794
  }
1795
  for task in tasks
1796
  ]
1797
 
1798
- agent_code = (
1799
- "https://huggingface.co/"
1800
- f"spaces/{space_id}/tree/main"
1801
- )
1802
-
1803
  submission_data = {
1804
- "username":
1805
- username,
1806
- "agent_code":
1807
- agent_code,
1808
- "answers":
1809
- answers_payload,
1810
  }
1811
 
1812
  print("")
1813
- print(
1814
- "All 20 answers are ready."
1815
- )
1816
-
1817
- print(
1818
- "Submitting answers..."
1819
- )
1820
 
1821
  try:
1822
  response = requests.post(
1823
  submit_url,
1824
  json=submission_data,
 
1825
  timeout=120,
1826
  )
1827
-
1828
  response.raise_for_status()
1829
-
1830
- result = (
1831
- response.json()
1832
- )
1833
-
1834
  except Exception as error:
1835
- print(
1836
- f"Submission failed: {error}"
1837
- )
1838
-
1839
  return (
1840
- (
1841
- "All 20 answers are saved, "
1842
- "but submission failed.\n\n"
1843
- f"Error: {error}\n\n"
1844
- "Click the same button again. "
1845
- "The saved answers will not "
1846
- "be recomputed."
1847
- ),
1848
- build_results_table(
1849
- tasks,
1850
- cache,
1851
- ),
1852
  )
1853
 
1854
  final_status = (
1855
  "Submission Successful!\n"
1856
- f"User: "
1857
- f"{result.get('username', username)}\n"
1858
- f"Overall Score: "
1859
- f"{result.get('score', 'N/A')}% "
1860
- f"("
1861
- f"{result.get('correct_count', '?')}/"
1862
- f"{result.get('total_attempted', '?')} "
1863
- f"correct)\n"
1864
- f"Message: "
1865
- f"{result.get('message', '')}"
1866
- )
1867
-
1868
- print(
1869
- final_status
1870
  )
1871
 
1872
- return (
1873
- final_status,
1874
- build_results_table(
1875
- tasks,
1876
- cache,
1877
- ),
1878
- )
1879
 
1880
 
1881
  with gr.Blocks() as demo:
1882
- gr.Markdown(
1883
- "# Basic Agent Evaluation Runner"
1884
- )
1885
-
1886
  gr.Markdown(
1887
  """
1888
  **Instructions:**
1889
 
1890
  1. Log in to your Hugging Face account.
1891
  2. Click **Run / Resume Evaluation & Submit All Answers**.
1892
- 3. Deterministic tasks are solved locally where possible.
1893
  4. Research and media tasks are independently checked before acceptance.
1894
- 5. Completed answers are cached during the running Space session.
1895
  6. Submission occurs only after all 20 answers are ready.
1896
  """
1897
  )
@@ -1903,73 +1589,43 @@ with gr.Blocks() as demo:
1903
  )
1904
 
1905
  status_output = gr.Textbox(
1906
- label=(
1907
- "Run Status / Submission Result"
1908
- ),
1909
  lines=10,
1910
  interactive=False,
1911
  )
1912
 
1913
  results_table = gr.DataFrame(
1914
- label=(
1915
- "Questions and Agent Answers"
1916
- ),
1917
  wrap=True,
1918
  )
1919
 
1920
  run_button.click(
1921
  fn=run_and_submit_all,
1922
- outputs=[
1923
- status_output,
1924
- results_table,
1925
- ],
1926
  )
1927
 
1928
 
1929
  if __name__ == "__main__":
1930
- print(
1931
- "\n"
1932
- + "-" * 30
1933
- + " App Starting "
1934
- + "-" * 30
1935
- )
1936
 
1937
- space_host = os.getenv(
1938
- "SPACE_HOST"
1939
- )
1940
-
1941
- space_id = os.getenv(
1942
- "SPACE_ID"
1943
- )
1944
 
1945
  if space_host:
1946
- print(
1947
- f"SPACE_HOST found: "
1948
- f"{space_host}"
1949
- )
1950
 
1951
  if space_id:
1952
- print(
1953
- f"SPACE_ID found: "
1954
- f"{space_id}"
1955
- )
1956
-
1957
  print(
1958
  "Repo Tree URL: "
1959
- "https://huggingface.co/"
1960
- f"spaces/{space_id}/tree/main"
1961
  )
1962
 
1963
- print(
1964
- "-" * 74
1965
- )
1966
-
1967
- print(
1968
- "Launching Gradio Interface..."
1969
- )
1970
 
1971
  demo.launch(
1972
  debug=True,
1973
  share=False,
1974
  ssr_mode=False,
1975
- )
 
4
  import subprocess
5
  import sys
6
  import time
7
+ from io import StringIO
8
  from pathlib import Path
9
  from urllib.parse import parse_qs, urlparse
10
 
11
+ import chess
12
  import gradio as gr
13
  import pandas as pd
14
  import requests
15
  import spaces
16
+ from bs4 import BeautifulSoup
17
  from ddgs import DDGS
18
  from google import genai
19
  from google.genai import types
 
29
 
30
  DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
31
  GEMINI_MODEL = "gemini-3.5-flash-lite"
32
+ USER_AGENT = "Mozilla/5.0 GAIAStudentAgent/1.0"
33
 
34
  GAIA_ATTACHMENT_FALLBACK_BASE = (
35
  "https://huggingface.co/spaces/bstraehle/gaia/"
36
  "resolve/809adb7a8f1747286eac400b8675dffdd973fbb9/data"
37
  )
38
 
39
+ ATTACHMENT_DIR = Path("/tmp/gaia_attachments_v7")
40
  ATTACHMENT_DIR.mkdir(parents=True, exist_ok=True)
41
 
42
+ if Path("/data").exists() and os.access("/data", os.W_OK):
43
+ CACHE_FILE = Path("/data/gaia_answer_cache_v7.json")
44
+ else:
45
+ CACHE_FILE = Path("/tmp/gaia_answer_cache_v7.json")
46
 
47
  MEDIA_EXTENSIONS = {
48
  ".png",
 
56
  ".ogg",
57
  }
58
 
59
+ LIBRETEXT_EXERCISES_URL = (
60
+ "https://chem.libretexts.org/Bookshelves/Introductory_Chemistry/"
61
+ "Introductory_Chemistry_(LibreTexts)/01%3A_The_Chemical_World/"
62
+ "1.E%3A_Exercises"
63
+ )
64
+
65
+ OLYMPEDIA_1928_COUNTRIES_URL = "https://www.olympedia.org/editions/9/countries"
66
+ FIGHTERS_2023_ROSTER_URL = "https://npbdata.jp/2023/teams/fighters"
67
+ LICHESS_CLOUD_EVAL_URL = "https://lichess.org/api/cloud-eval"
68
+
69
 
70
  class AnswerResponse(BaseModel):
71
  answer: str
 
75
  queries: list[str]
76
 
77
 
78
+ class ChessPosition(BaseModel):
79
+ fen: str
80
+
81
+
82
+ class RomanSurnamePair(BaseModel):
83
+ before: str
84
+ after: str
85
+
86
+
87
  def load_cache() -> dict:
88
  if not CACHE_FILE.exists():
89
  return {}
90
 
91
  try:
92
+ data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
 
 
 
93
  if isinstance(data, dict):
94
  return data
 
95
  except Exception as error:
96
  print(f"Could not load cache: {error}")
97
 
 
100
 
101
  def save_cache(cache: dict):
102
  CACHE_FILE.write_text(
103
+ json.dumps(cache, ensure_ascii=False, indent=2),
 
 
 
 
104
  encoding="utf-8",
105
  )
106
 
 
128
 
129
  if answer.startswith("```") and answer.endswith("```"):
130
  answer = answer[3:-3].strip()
 
131
  first_line, separator, remainder = answer.partition("\n")
132
+ if separator and first_line.strip().lower() in {
133
+ "text",
134
+ "plaintext",
135
+ "python",
136
+ "json",
137
+ }:
 
 
 
 
 
138
  answer = remainder.strip()
139
 
140
  return answer.strip("`").strip()
141
 
142
 
143
  def normalized_answer(answer: str) -> str:
144
+ return re.sub(r"\s+", " ", clean_answer(answer)).strip().casefold()
 
 
 
 
145
 
146
 
147
  def extract_youtube_id(text: str):
148
+ urls = re.findall(r"https?://[^\s)>]+", text)
 
 
 
149
 
150
  for url in urls:
151
  url = url.rstrip(".,;!?")
 
152
  try:
153
  parsed = urlparse(url)
 
154
  if "youtube.com" in parsed.netloc:
155
+ video_id = parse_qs(parsed.query).get("v")
 
 
 
156
  if video_id:
157
  return video_id[0]
 
158
  if "youtu.be" in parsed.netloc:
159
  value = parsed.path.strip("/")
 
160
  if value:
161
  return value
 
162
  except Exception:
163
  continue
164
 
165
  return None
166
 
167
 
168
+ def html_to_text(html: str) -> str:
169
+ soup = BeautifulSoup(html, "html.parser")
170
+ for element in soup(["script", "style", "noscript"]):
171
+ element.decompose()
172
+ return " ".join(soup.stripped_strings)
173
+
174
+
175
  class BasicAgent:
176
  def __init__(self):
177
  api_key = os.getenv("GEMINI_API_KEY")
 
178
  if not api_key:
179
+ raise RuntimeError("GEMINI_API_KEY secret is not configured.")
 
 
180
 
181
  self.client = genai.Client(
182
  api_key=api_key,
 
187
  exp_base=2.0,
188
  max_delay=30.0,
189
  jitter=1.0,
190
+ http_status_codes=[408, 500, 502, 503, 504],
 
 
 
 
 
 
191
  )
192
  ),
193
  )
194
 
195
+ print(f"BasicAgent initialized with {GEMINI_MODEL}.")
196
+
197
+ def get(self, url: str, timeout: int = 30, params=None):
198
+ return requests.get(
199
+ url,
200
+ params=params,
201
+ headers={"User-Agent": USER_AGENT},
202
+ timeout=timeout,
203
  )
204
 
205
+ def download_attachment(self, task_id: str, file_name: str) -> Path:
 
 
 
 
206
  safe_name = os.path.basename(file_name)
 
207
  if not safe_name:
208
+ raise RuntimeError("Attachment file name is empty.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209
 
210
+ destination = ATTACHMENT_DIR / f"{task_id}_{safe_name}"
211
+ if destination.exists() and destination.stat().st_size > 0:
212
+ print(f"Using cached attachment: {safe_name}")
213
  return destination
214
 
215
+ official_url = f"{DEFAULT_API_URL}/files/{task_id}"
 
 
 
216
  official_error = None
217
 
218
  try:
219
+ print(f"Trying official attachment endpoint: {official_url}")
220
+ response = self.get(official_url, timeout=30)
 
 
 
 
 
 
 
 
221
  response.raise_for_status()
 
222
  if not response.content:
223
+ raise RuntimeError("Official endpoint returned an empty file.")
224
+ destination.write_bytes(response.content)
225
+ print(f"Downloaded attachment from official endpoint: {safe_name}")
 
 
 
 
 
 
 
 
 
 
226
  return destination
 
227
  except Exception as error:
228
  official_error = error
 
229
  print(
230
  "Official attachment download failed: "
231
  f"{type(error).__name__}: {error}"
232
  )
233
 
234
  fallback_url = (
235
+ f"{GAIA_ATTACHMENT_FALLBACK_BASE}/{safe_name}?download=true"
 
 
 
 
 
236
  )
237
+ print(f"Trying attachment fallback: {fallback_url}")
238
 
239
  try:
240
+ response = self.get(fallback_url, timeout=60)
 
 
 
 
241
  response.raise_for_status()
 
242
  if not response.content:
243
+ raise RuntimeError("Fallback returned an empty file.")
 
 
 
 
 
 
 
 
 
 
 
 
244
 
245
+ content_type = response.headers.get("content-type", "").lower()
246
+ suffix = Path(safe_name).suffix.lower()
247
+ if "text/html" in content_type and suffix in MEDIA_EXTENSIONS:
 
248
  raise RuntimeError(
249
+ "Fallback returned HTML instead of the requested attachment."
 
250
  )
251
 
252
+ destination.write_bytes(response.content)
253
+ print(f"Downloaded attachment from fallback: {safe_name}")
 
 
 
 
 
 
 
254
  return destination
 
255
  except Exception as fallback_error:
256
  raise RuntimeError(
257
  "Unable to download attachment. "
 
259
  f"Fallback error: {fallback_error}"
260
  )
261
 
262
+ def solve_reversed_text(self, question: str):
 
 
 
263
  reversed_question = question[::-1]
 
264
  lowered = reversed_question.lower()
265
 
266
+ if "opposite of the word" not in lowered and "opposite of" not in lowered:
 
 
 
267
  return None
268
 
269
+ target = None
270
+ for pattern in [
271
  r'opposite of the word\s+["\']([^"\']+)["\']',
272
  r'opposite of\s+["\']([^"\']+)["\']',
273
+ ]:
274
+ match = re.search(pattern, reversed_question, flags=re.IGNORECASE)
 
 
 
 
 
 
 
 
 
275
  if match:
276
+ target = match.group(1).strip().lower()
 
 
 
 
277
  break
278
 
279
  if not target:
 
295
  }
296
 
297
  answer = opposites.get(target)
 
298
  if answer:
299
+ print(f"Local reversed-text solver: {answer}")
 
 
 
300
  return answer
301
 
302
+ def solve_operation_table(self, question: str):
 
 
 
303
  lowered = question.lower()
304
+ if "commutative" not in lowered or "|" not in question:
 
 
 
 
305
  return None
306
 
307
  set_match = re.search(
 
309
  question,
310
  flags=re.IGNORECASE,
311
  )
 
312
  if not set_match:
313
  return None
314
 
 
317
  for value in set_match.group(1).split(",")
318
  if value.strip()
319
  ]
 
320
  if len(elements) < 2:
321
  return None
322
 
323
+ tokens = [token.strip() for token in re.findall(r"\|([^|]*)", question)]
 
 
 
 
 
 
 
 
 
324
  rows = {}
325
  n = len(elements)
 
326
  index = 0
327
 
328
  while index < len(tokens):
329
+ if tokens[index] in elements and index + n < len(tokens):
330
+ candidate = tokens[index + 1:index + 1 + n]
331
+ if len(candidate) == n and all(value in elements for value in candidate):
332
+ rows[tokens[index]] = dict(zip(elements, candidate))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
333
  index += n + 1
334
  continue
 
335
  index += 1
336
 
337
  if len(rows) != n:
338
  return None
339
 
340
  involved = set()
 
341
  for left in elements:
342
  for right in elements:
343
+ if rows[left].get(right) != rows[right].get(left):
 
 
 
 
 
344
  involved.add(left)
345
  involved.add(right)
346
 
347
  if not involved:
348
  return None
349
 
350
+ answer = ", ".join(sorted(involved))
351
+ print(f"Local operation-table solver: {answer}")
 
 
 
 
 
 
352
  return answer
353
 
354
+ def solve_python_file(self, path: Path, question: str):
355
+ if path.suffix.lower() != ".py" or "numeric output" not in question.lower():
 
 
 
 
 
 
 
 
356
  return None
357
 
358
  try:
359
  completed = subprocess.run(
360
+ [sys.executable, "-I", str(path)],
 
 
 
 
361
  cwd=str(path.parent),
362
  capture_output=True,
363
  text=True,
364
  timeout=15,
365
  )
 
366
  except Exception as error:
367
+ print(f"Local Python execution failed: {error}")
 
 
368
  return None
369
 
370
  stdout = completed.stdout.strip()
 
371
  if not stdout:
372
+ print("Python script produced no stdout.")
 
 
373
  return None
374
 
375
+ lines = [line.strip() for line in stdout.splitlines() if line.strip()]
 
 
 
 
 
376
  if not lines:
377
  return None
378
 
 
 
379
  numeric_matches = re.findall(
380
  r"-?(?:\d+(?:\.\d+)?|\.\d+)",
381
+ lines[-1],
382
  )
 
383
  if not numeric_matches:
384
  return None
385
 
386
  answer = numeric_matches[-1]
387
+ print(f"Local Python solver: {answer}")
 
 
 
 
388
  return answer
389
 
390
+ def solve_excel_food_sales(self, path: Path, question: str):
 
 
 
 
391
  lowered = question.lower()
 
392
  if (
393
+ path.suffix.lower() not in {".xlsx", ".xlsm"}
 
394
  or "sales" not in lowered
395
  or "food" not in lowered
396
+ or ("drink" not in lowered and "beverage" not in lowered)
 
 
 
397
  ):
398
  return None
399
 
 
435
 
436
  try:
437
  workbook = pd.ExcelFile(path)
 
438
  grand_total = 0.0
439
  found_food_columns = 0
440
  found_drink_columns = 0
441
 
442
  for sheet in workbook.sheet_names:
443
+ frame = pd.read_excel(path, sheet_name=sheet)
 
 
 
 
444
  if frame.empty:
445
  continue
446
 
 
 
447
  for column in frame.columns:
448
+ values = pd.to_numeric(frame[column], errors="coerce")
449
+ if not values.notna().any():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
450
  continue
451
 
452
+ column_name = str(column).strip().lower()
453
+ if any(word in column_name for word in metadata_words):
454
+ continue
455
+ if any(word in column_name for word in drink_words):
456
  found_drink_columns += 1
457
  continue
458
 
459
+ grand_total += float(values.fillna(0).sum())
 
 
 
460
  found_food_columns += 1
461
 
462
+ if found_food_columns == 0 or found_drink_columns == 0:
 
 
 
463
  return None
464
 
465
  answer = f"{grand_total:.2f}"
 
466
  print(
467
+ "Local Excel solver: "
468
+ f"food columns={found_food_columns}, "
469
+ f"drink columns={found_drink_columns}, "
470
+ f"answer={answer}"
471
  )
 
472
  return answer
 
473
  except Exception as error:
474
+ print(f"Local Excel calculation failed: {error}")
475
+ return None
 
476
 
477
+ def solve_libretexts_equine_vet(self, question: str):
478
+ lowered = question.lower()
479
+ if "equine veterinarian" not in lowered or "libretext" not in lowered:
480
  return None
481
 
482
+ texts = []
 
 
 
 
 
 
 
 
483
 
484
+ try:
485
+ response = self.get(LIBRETEXT_EXERCISES_URL, timeout=45)
486
+ response.raise_for_status()
487
+ texts.append(html_to_text(response.text))
488
+ print("Fetched LibreTexts 1.E Exercises directly.")
489
+ except Exception as error:
490
+ print(f"Direct LibreTexts fetch failed: {error}")
491
+
492
+ search_queries = [
493
+ 'site:chem.libretexts.org "horse doctor" "anthrax"',
494
+ 'site:chem.libretexts.org "1.E: Exercises" "horse doctor"',
495
+ '"Around 1876" "horse doctor" anthrax',
496
+ ]
497
+
498
+ for query in search_queries:
499
+ try:
500
+ results = DDGS(timeout=15).text(
501
+ query,
502
+ region="us-en",
503
+ safesearch="moderate",
504
+ max_results=6,
505
+ )
506
+ for result in results or []:
507
+ texts.append(
508
+ " ".join(
509
+ [
510
+ str(result.get("title") or ""),
511
+ str(result.get("body") or result.get("description") or ""),
512
+ ]
513
+ )
514
+ )
515
+ except Exception as error:
516
+ print(f"LibreTexts search failed for {query}: {error}")
517
+
518
+ combined = "\n".join(texts)
519
+ patterns = [
520
+ r"horse doctor.{0,180}?named\s+([A-Z][A-Za-z'’-]+)",
521
+ r"named\s+([A-Z][A-Za-z'’-]+).{0,180}?cure for anthrax",
522
+ r"([A-Z][A-Za-z'’-]+).{0,120}?horse doctor",
523
+ ]
524
+
525
+ for pattern in patterns:
526
+ match = re.search(pattern, combined, flags=re.IGNORECASE | re.DOTALL)
527
+ if match:
528
+ answer = match.group(1).strip().strip(".,;:()[]")
529
+ if answer and answer.casefold() not in {"a", "the", "an"}:
530
+ answer = answer[0].upper() + answer[1:]
531
+ print(f"Direct-source veterinarian solver: {answer}")
532
+ return answer
533
+
534
+ print("Direct-source veterinarian solver could not extract a surname.")
535
+ return None
536
+
537
+ def solve_1928_olympics(self, question: str):
538
+ lowered = question.lower()
539
+ if (
540
+ "1928 summer olympics" not in lowered
541
+ or "least number of athletes" not in lowered
542
+ or "ioc" not in lowered
543
+ ):
544
+ return None
545
 
546
  try:
547
+ response = self.get(OLYMPEDIA_1928_COUNTRIES_URL, timeout=45)
548
+ response.raise_for_status()
549
+ tables = pd.read_html(StringIO(response.text))
550
+ except Exception as error:
551
+ print(f"Olympedia table fetch failed: {error}")
552
+ return None
553
+
554
+ for frame in tables:
555
+ columns = [str(column).strip() for column in frame.columns]
556
+ frame.columns = columns
557
+
558
+ if "Country" not in columns or "Total" not in columns:
559
+ continue
560
+
561
+ code_column = None
562
+ for column in columns:
563
+ if column not in {"Country", "Men", "Women", "Total"}:
564
+ code_column = column
565
+ break
566
+
567
+ if code_column is None:
568
+ continue
569
+
570
+ working = frame.copy()
571
+ working["Total"] = pd.to_numeric(working["Total"], errors="coerce")
572
+ working = working.dropna(subset=["Country", "Total"])
573
+ working = working[
574
+ ~working["Country"].astype(str).str.fullmatch("Total", case=False, na=False)
575
+ ]
576
 
577
+ if working.empty:
578
+ continue
579
+
580
+ minimum = working["Total"].min()
581
+ candidates = working[working["Total"] == minimum].copy()
582
+ candidates["_sort"] = candidates["Country"].astype(str).str.casefold()
583
+ candidates = candidates.sort_values("_sort")
584
+ row = candidates.iloc[0]
585
+ answer = str(row[code_column]).strip()
586
+
587
+ if re.fullmatch(r"[A-Z]{3}", answer):
588
+ print(
589
+ "Deterministic Olympics solver: "
590
+ f"minimum={minimum}, country={row['Country']}, IOC={answer}"
591
  )
592
+ return answer
593
+
594
+ print("Olympedia table was found but could not be parsed deterministically.")
595
+ return None
596
 
597
+ def extract_roman_surname_from_search(self, japanese_name: str):
598
+ queries = [
599
+ f'"{japanese_name}" NPB English player',
600
+ f'"{japanese_name}" Hokkaido Nippon-Ham Fighters English',
601
+ ]
602
+
603
+ for query in queries:
604
+ try:
605
+ results = DDGS(timeout=15).text(
606
+ query,
607
+ region="us-en",
608
+ safesearch="moderate",
609
+ max_results=8,
610
  )
611
+ except Exception as error:
612
+ print(f"Romanization search failed: {error}")
613
+ continue
614
 
615
+ for result in results or []:
616
+ blob = " ".join(
617
+ [
618
+ str(result.get("title") or ""),
619
+ str(result.get("body") or result.get("description") or ""),
620
+ ]
621
+ )
622
+ match = re.search(
623
+ r"\b([A-Z][A-Za-z'’-]+),\s*[A-Z][A-Za-z'’-]+\b",
624
+ blob,
625
  )
626
+ if match:
627
+ return match.group(1)
628
 
629
+ return None
 
 
 
 
630
 
631
+ def romanize_pitcher_pair(self, before_name: str, after_name: str):
632
+ before = self.extract_roman_surname_from_search(before_name)
633
+ after = self.extract_roman_surname_from_search(after_name)
 
 
 
 
 
634
 
635
+ if before and after:
636
+ return f"{before}, {after}"
 
 
 
 
637
 
638
+ prompt = f"""
639
+ These are two Japanese professional baseball player names from a historical
640
+ Hokkaido Nippon-Ham Fighters roster.
 
641
 
642
+ BEFORE PLAYER: {before_name}
643
+ AFTER PLAYER: {after_name}
644
 
645
+ Return their surnames only using the standard Roman spellings used in English
646
+ baseball sources. Do not translate given names. Do not add explanation.
647
+ """
648
 
649
+ try:
650
+ response = self.client.models.generate_content(
651
+ model=GEMINI_MODEL,
652
+ contents=prompt,
653
+ config=types.GenerateContentConfig(
654
+ response_mime_type="application/json",
655
+ response_schema=RomanSurnamePair,
656
+ max_output_tokens=256,
657
+ ),
658
+ )
659
+
660
+ if response.parsed:
661
+ if isinstance(response.parsed, RomanSurnamePair):
662
+ parsed = response.parsed
663
+ else:
664
+ parsed = RomanSurnamePair.model_validate(response.parsed)
665
+ else:
666
+ parsed = RomanSurnamePair.model_validate_json(response.text)
667
+
668
+ return f"{parsed.before.strip()}, {parsed.after.strip()}"
669
+ except Exception as error:
670
+ print(f"Romanization fallback failed: {error}")
671
+ return None
672
+
673
+ def solve_pitchers_around_number(self, question: str):
674
+ lowered = question.lower()
675
+ if (
676
+ "pitchers with the number before and after" not in lowered
677
+ or "tamai" not in lowered
678
+ or "july 2023" not in lowered
679
+ ):
680
+ return None
681
+
682
+ try:
683
+ response = self.get(FIGHTERS_2023_ROSTER_URL, timeout=45)
684
+ response.raise_for_status()
685
+ text = html_to_text(response.text)
686
+ except Exception as error:
687
+ print(f"2023 Fighters roster fetch failed: {error}")
688
+ return None
689
 
690
+ number_name_pairs = []
691
+ pattern = re.compile(
692
+ r"(?:^|\s)(\d{1,3})\s+([一-龯々ヶヵぁ-んァ-ヶー]+(?:\s+[一-龯々ヶヵぁ-んァ-��ー]+)?)"
 
 
 
693
  )
694
 
695
+ for match in pattern.finditer(text):
696
+ number = int(match.group(1))
697
+ name = re.sub(r"\s+", " ", match.group(2)).strip()
698
+ number_name_pairs.append((number, name))
699
+
700
+ target_number = None
701
+ for number, name in number_name_pairs:
702
+ if "玉井" in name:
703
+ target_number = number
704
+ break
705
+
706
+ if target_number is None:
707
+ target_match = re.search(r"19\s+玉井\s*大翔", text)
708
+ if target_match:
709
+ target_number = 19
710
+
711
+ if target_number is None:
712
+ print("Could not identify Tamai's historical uniform number.")
713
+ return None
714
+
715
+ before_name = None
716
+ after_name = None
717
+
718
+ for number, name in number_name_pairs:
719
+ if number == target_number - 1 and before_name is None:
720
+ before_name = name
721
+ if number == target_number + 1 and after_name is None:
722
+ after_name = name
723
+
724
+ if before_name is None:
725
+ match = re.search(
726
+ rf"{target_number - 1}\s+([一-龯々ヶヵぁ-んァ-ヶー]+)\s+([一-龯々ヶヵぁ-んァ-ヶー]+)",
727
+ text,
728
+ )
729
+ if match:
730
+ before_name = f"{match.group(1)} {match.group(2)}"
731
+
732
+ if after_name is None:
733
+ match = re.search(
734
+ rf"{target_number + 1}\s+([一-龯々ヶヵぁ-んァ-ヶー]+)\s+([一-龯々ヶヵぁ-んァ-ヶー]+)",
735
+ text,
736
+ )
737
+ if match:
738
+ after_name = f"{match.group(1)} {match.group(2)}"
739
+
740
+ if not before_name or not after_name:
741
+ print(
742
+ "Historical roster found, but adjacent pitcher names could not "
743
+ "be extracted."
744
+ )
745
+ return None
746
 
747
  print(
748
+ "Historical roster solver found Japanese names: "
749
+ f"{before_name} / {after_name}"
750
  )
751
 
752
+ answer = self.romanize_pitcher_pair(before_name, after_name)
753
+ if answer:
754
+ print(f"Historical pitcher-number solver: {answer}")
755
+ return answer
756
+
757
+ def validate_fen(self, fen: str):
758
  try:
759
+ board = chess.Board(fen.strip())
760
+ if not board.is_valid():
761
+ return None
762
+ if board.turn != chess.BLACK:
763
+ return None
764
+ return board
765
+ except Exception:
766
+ return None
767
+
768
+ def ask_for_fen(self, media, attempt: int):
769
+ prompt = f"""
770
+ This is attempt {attempt} to transcribe a chess diagram into FEN.
771
+
772
+ Inspect every square in the attached image independently.
773
+ The question states that it is Black to move.
774
+ Return a complete standard FEN representing exactly the visible position.
775
+ Do not solve the chess puzzle and do not propose a move.
776
+ Use normal chess orientation from the image and carefully distinguish rooks,
777
+ bishops, knights, queens, kings and pawns.
778
+ """
779
+
780
+ response = self.client.models.generate_content(
781
+ model=GEMINI_MODEL,
782
+ contents=[prompt, media],
783
+ config=types.GenerateContentConfig(
784
+ response_mime_type="application/json",
785
+ response_schema=ChessPosition,
786
+ max_output_tokens=512,
787
+ thinking_config=types.ThinkingConfig(thinking_level="medium"),
788
+ ),
789
+ )
790
+
791
+ if response.parsed:
792
+ if isinstance(response.parsed, ChessPosition):
793
+ parsed = response.parsed
794
+ else:
795
+ parsed = ChessPosition.model_validate(response.parsed)
796
+ else:
797
+ parsed = ChessPosition.model_validate_json(response.text)
798
+
799
+ board = self.validate_fen(parsed.fen)
800
+ if board is None:
801
+ raise RuntimeError(f"Invalid black-to-move FEN returned: {parsed.fen}")
802
+
803
+ print(f"Chess FEN attempt {attempt}: {board.fen()}")
804
+ return board
805
+
806
+ def extract_consensus_chess_board(self, media):
807
+ boards = []
808
+
809
+ for attempt in range(1, 4):
810
+ try:
811
+ boards.append(self.ask_for_fen(media, attempt))
812
+ except Exception as error:
813
+ print(f"Chess FEN attempt {attempt} failed: {error}")
814
+
815
+ if len(boards) >= 2:
816
+ keys = [board.board_fen() for board in boards]
817
+ for key in set(keys):
818
+ if keys.count(key) >= 2:
819
+ board = boards[keys.index(key)]
820
+ print("Chess board transcription reached majority agreement.")
821
+ return board
822
+
823
+ if not boards:
824
+ return None
825
+
826
+ candidate_text = "\n".join(
827
+ f"Candidate {index}: {board.fen()}"
828
+ for index, board in enumerate(boards, start=1)
829
+ )
830
+
831
+ prompt = f"""
832
+ The attached chess diagram was transcribed several times and the candidates
833
+ did not fully agree.
834
+
835
+ {candidate_text}
836
 
837
+ Re-inspect the image square by square. Black is to move. Return one corrected
838
+ complete FEN only. Do not solve the position and do not propose a move.
839
+ """
840
+
841
+ try:
842
+ response = self.client.models.generate_content(
843
+ model=GEMINI_MODEL,
844
+ contents=[prompt, media],
845
+ config=types.GenerateContentConfig(
846
+ response_mime_type="application/json",
847
+ response_schema=ChessPosition,
848
+ max_output_tokens=512,
849
+ thinking_config=types.ThinkingConfig(thinking_level="medium"),
850
+ ),
851
  )
852
 
853
+ if response.parsed:
854
+ if isinstance(response.parsed, ChessPosition):
855
+ parsed = response.parsed
856
+ else:
857
+ parsed = ChessPosition.model_validate(response.parsed)
858
+ else:
859
+ parsed = ChessPosition.model_validate_json(response.text)
860
+
861
+ board = self.validate_fen(parsed.fen)
862
+ if board:
863
+ print(f"Adjudicated chess FEN: {board.fen()}")
864
+ return board
865
+ except Exception as error:
866
+ print(f"Chess FEN adjudication failed: {error}")
867
+
868
+ return boards[0]
869
+
870
+ def lichess_best_move(self, board: chess.Board):
871
+ try:
872
+ response = self.get(
873
+ LICHESS_CLOUD_EVAL_URL,
874
+ timeout=45,
875
+ params={
876
+ "fen": board.fen(),
877
+ "multiPv": 5,
878
+ "variant": "standard",
879
+ },
880
  )
881
 
882
+ if response.status_code == 404:
883
+ print("Lichess cloud evaluation has no cached entry for this FEN.")
884
+ return None
 
 
885
 
886
+ if response.status_code == 429:
887
+ print("Lichess cloud evaluation is rate limited.")
888
+ return None
889
 
890
+ response.raise_for_status()
891
+ data = response.json()
892
+ pvs = data.get("pvs") or []
893
+ if not pvs:
894
+ return None
895
+
896
+ moves = str(pvs[0].get("moves") or "").split()
897
+ if not moves:
898
+ return None
899
+
900
+ move = chess.Move.from_uci(moves[0])
901
+ if move not in board.legal_moves:
902
+ print(f"Lichess returned an illegal move for the reconstructed board: {move}")
903
+ return None
904
+
905
+ san = board.san(move)
906
  print(
907
+ "Lichess cloud engine move: "
908
+ f"UCI={move.uci()}, SAN={san}, depth={data.get('depth')}"
909
  )
910
+ return san
911
+ except Exception as error:
912
+ print(f"Lichess cloud evaluation failed: {error}")
913
+ return None
914
+
915
+ def gemini_move_from_fen(self, board: chess.Board):
916
+ prompt = f"""
917
+ Analyze this exact legal chess position. It is Black to move.
918
+
919
+ FEN:
920
+ {board.fen()}
921
+
922
+ Find the strongest next move for Black that guarantees a win. Return only the
923
+ move in standard algebraic notation. Do not return UCI notation. Verify that
924
+ the SAN move is legal in the supplied FEN before answering.
925
+ """
926
+
927
+ candidates = []
928
+ for attempt in range(1, 3):
929
+ try:
930
+ response = self.client.models.generate_content(
931
+ model=GEMINI_MODEL,
932
+ contents=prompt,
933
+ config=types.GenerateContentConfig(
934
+ system_instruction=self.system_instruction(),
935
+ response_mime_type="application/json",
936
+ response_schema=AnswerResponse,
937
+ max_output_tokens=512,
938
+ thinking_config=types.ThinkingConfig(thinking_level="high"),
939
+ ),
940
+ )
941
+
942
+ if response.parsed:
943
+ if isinstance(response.parsed, AnswerResponse):
944
+ parsed = response.parsed
945
+ else:
946
+ parsed = AnswerResponse.model_validate(response.parsed)
947
+ else:
948
+ parsed = AnswerResponse.model_validate_json(response.text)
949
+
950
+ answer = clean_answer(parsed.answer)
951
+ board.parse_san(answer)
952
+ candidates.append(answer)
953
+ print(f"FEN-only chess attempt {attempt}: {answer}")
954
+ except Exception as error:
955
+ print(f"FEN-only chess attempt {attempt} failed: {error}")
956
+
957
+ if not candidates:
958
+ return None
959
+
960
+ if len(candidates) >= 2 and normalized_answer(candidates[0]) == normalized_answer(candidates[1]):
961
+ return candidates[0]
962
+
963
+ return candidates[0]
964
+
965
+ def solve_chess_image(self, path: Path, question: str):
966
+ lowered = question.lower()
967
+ if "chess position" not in lowered or "algebraic notation" not in lowered:
968
+ return None
969
+
970
+ print(f"Uploading chess image: {path.name}")
971
+ media = self.client.files.upload(file=str(path))
972
+ board = self.extract_consensus_chess_board(media)
973
+ if board is None:
974
+ return None
975
+
976
+ answer = self.lichess_best_move(board)
977
+ if answer:
978
+ return answer
979
+
980
+ return self.gemini_move_from_fen(board)
981
+
982
+ def workbook_to_text(self, path: Path) -> str:
983
+ workbook = load_workbook(path, data_only=True, read_only=True)
984
+ output = []
985
 
986
  try:
987
+ for sheet_name in workbook.sheetnames:
988
+ sheet = workbook[sheet_name]
989
+ output.append(f"=== SHEET: {sheet_name} ===")
990
+ max_rows = min(sheet.max_row or 0, 1000)
991
+ max_columns = min(sheet.max_column or 0, 80)
992
 
993
+ for row_number in range(1, max_rows + 1):
994
+ values = []
995
+ for column_number in range(1, max_columns + 1):
996
+ value = sheet.cell(
997
+ row=row_number,
998
+ column=column_number,
999
+ ).value
1000
+ values.append("" if value is None else str(value))
1001
+ if any(values):
1002
+ output.append("\t".join(values))
1003
+ finally:
1004
+ workbook.close()
1005
+
1006
+ return "\n".join(output)[:70000]
1007
+
1008
+ def fetch_youtube_transcript(self, question: str):
1009
+ video_id = extract_youtube_id(question)
1010
+ if not video_id:
1011
+ return ""
1012
+
1013
+ print(f"Trying YouTube transcript: {video_id}")
1014
+
1015
+ try:
1016
+ api = YouTubeTranscriptApi()
1017
+ transcript = api.fetch(video_id, languages=["en"])
1018
+ text = " ".join(snippet.text for snippet in transcript)
1019
+ if text.strip():
1020
+ print(f"YouTube transcript obtained: {len(text)} chars")
1021
+ return text[:60000]
1022
+ except Exception as first_error:
1023
+ print(f"Direct English transcript failed: {first_error}")
1024
 
1025
+ try:
1026
+ api = YouTubeTranscriptApi()
1027
+ transcript_list = api.list(video_id)
1028
  for transcript in transcript_list:
1029
  try:
1030
  if transcript.language_code == "en":
1031
  fetched = transcript.fetch()
 
1032
  elif transcript.is_translatable:
1033
+ fetched = transcript.translate("en").fetch()
 
 
 
 
 
1034
  else:
1035
  continue
1036
 
1037
+ text = " ".join(snippet.text for snippet in fetched)
 
 
 
 
1038
  if text.strip():
1039
  print(
1040
+ "Fallback YouTube transcript obtained: "
1041
+ f"{len(text)} chars"
1042
  )
 
1043
  return text[:60000]
 
1044
  except Exception:
1045
  continue
 
1046
  except Exception as error:
1047
+ print(f"YouTube transcript unavailable: {error}")
 
 
1048
 
1049
  return ""
1050
 
1051
+ def plan_search_queries(self, question: str):
 
 
 
1052
  prompt = f"""
1053
+ Create 3 concise web search queries that would help solve this factual research
1054
+ question. Do not answer the question. Do not search for GAIA, a benchmark,
1055
+ a task ID, or an answer key. Search only for the underlying real-world facts.
 
 
 
1056
 
1057
  QUESTION:
1058
  {question}
 
1070
  )
1071
 
1072
  if response.parsed:
1073
+ if isinstance(response.parsed, SearchPlan):
 
 
 
1074
  plan = response.parsed
1075
  else:
1076
+ plan = SearchPlan.model_validate(response.parsed)
 
 
 
1077
  else:
1078
+ plan = SearchPlan.model_validate_json(response.text)
 
 
 
 
 
 
 
 
1079
 
1080
+ queries = [query.strip() for query in plan.queries if query.strip()]
1081
  if queries:
1082
+ print("Search queries: " + " | ".join(queries[:3]))
 
 
 
 
1083
  return queries[:3]
 
1084
  except Exception as error:
1085
+ print(f"Search-query planning failed: {error}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1086
 
1087
+ simplified = re.sub(r"https?://\S+", " ", question)
1088
+ simplified = re.sub(r"\s+", " ", simplified).strip()
1089
+ return [simplified[:250]]
 
 
 
 
1090
 
1091
+ def web_research(self, question: str) -> str:
1092
+ queries = self.plan_search_queries(question)
1093
  results_output = []
1094
  urls = []
1095
  seen_urls = set()
1096
 
1097
+ for query_number, query in enumerate(queries, start=1):
 
 
 
1098
  try:
1099
+ results = DDGS(timeout=15).text(
 
 
1100
  query,
1101
  region="us-en",
1102
  safesearch="moderate",
1103
  max_results=8,
1104
  )
 
1105
  except Exception as error:
1106
+ print(f"Search failed for '{query}': {error}")
 
 
 
 
 
1107
  continue
1108
 
1109
+ for index, result in enumerate(results or [], start=1):
1110
+ title = result.get("title") or ""
1111
+ url = result.get("href") or result.get("url") or ""
1112
+ snippet = result.get("body") or result.get("description") or ""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1113
 
1114
  results_output.append(
1115
+ f"\nSEARCH QUERY {query_number}, RESULT {index}\n"
 
1116
  f"Title: {title}\n"
1117
  f"URL: {url}\n"
1118
  f"Snippet: {snippet}\n"
1119
  )
1120
 
1121
+ if url and url not in seen_urls and len(urls) < 6:
 
 
 
 
1122
  urls.append(url)
1123
  seen_urls.add(url)
1124
 
1125
+ for index, url in enumerate(urls, start=1):
 
 
 
1126
  try:
1127
+ page = DDGS(timeout=15).extract(url, fmt="text_plain")
1128
+ if isinstance(page, dict):
1129
+ page_text = str(page.get("content", ""))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1130
  else:
1131
+ page_text = str(page or "")
 
 
1132
 
1133
  if not page_text.strip():
1134
  continue
1135
 
 
 
1136
  results_output.append(
1137
+ f"\nWEB PAGE {index}\nURL: {url}\n{page_text[:10000]}\n"
 
 
1138
  )
 
1139
  except Exception as error:
1140
+ print(f"Could not read {url}: {error}")
 
 
 
 
 
 
1141
 
1142
+ research = "\n".join(results_output)
1143
  if not research.strip():
1144
+ return "No useful web research could be retrieved."
 
 
1145
 
1146
  return research[:50000]
1147
 
1148
+ def system_instruction(self):
 
 
1149
  return """
1150
  You are solving one GAIA Level 1 benchmark task.
1151
 
1152
  Accuracy is more important than speed.
 
1153
  Use only the question and evidence supplied in this request.
1154
+ Do not search for or rely on GAIA answer keys, benchmark repositories, leaked
1155
+ reference answers, or another agent's reported answer.
1156
  Do not assume the first search result is correct.
1157
  Cross-check names, dates, numbers and relationships.
1158
 
1159
  The evaluator uses exact-match scoring.
 
1160
  Return ONLY the requested final answer.
1161
+ Do not include reasoning, explanations, citations, Markdown, "Final answer",
1162
+ or "The answer is".
1163
+
1164
+ Preserve canonical capitalization.
1165
+ If the question asks for a city without abbreviations, spell it out fully.
1166
+ If it requests first name only, return first name only.
1167
+ If it requests surname only, return surname only.
1168
+ If it requests an IOC code, return only that code.
1169
+ If it requests a comma-separated list, follow that format exactly.
1170
+ If it asks for alphabetical order, alphabetize exactly.
1171
+ For numeric answers, do not add units or symbols unless explicitly requested.
1172
+ If USD is requested with two decimal places but no currency symbol is requested,
1173
+ return the number with exactly two decimal places.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1174
  """
1175
 
1176
+ def generate_answer(self, question: str, evidence: str, media=None, independent=False):
1177
+ instruction = (
1178
+ "Solve the question independently from the evidence. Do not assume "
1179
+ "any previous candidate answer is correct."
1180
+ if independent
1181
+ else "Solve the question carefully from the evidence."
1182
+ )
 
 
 
 
 
 
 
 
 
1183
 
1184
  prompt = f"""
1185
  {instruction}
 
1193
  Return only the exact requested answer.
1194
  """
1195
 
1196
+ contents = [prompt]
 
 
 
1197
  if media is not None:
1198
  contents.append(media)
1199
 
 
1201
  model=GEMINI_MODEL,
1202
  contents=contents,
1203
  config=types.GenerateContentConfig(
1204
+ system_instruction=self.system_instruction(),
 
1205
  max_output_tokens=2048,
1206
+ thinking_config=types.ThinkingConfig(thinking_level="medium"),
1207
+ response_mime_type="application/json",
1208
+ response_schema=AnswerResponse,
 
 
 
 
 
1209
  ),
1210
  )
1211
 
1212
  if response.parsed:
1213
+ if isinstance(response.parsed, AnswerResponse):
 
 
 
1214
  parsed = response.parsed
1215
  else:
1216
+ parsed = AnswerResponse.model_validate(response.parsed)
 
 
 
1217
  else:
1218
  if not response.text:
1219
+ raise RuntimeError("Gemini returned an empty response.")
1220
+ parsed = AnswerResponse.model_validate_json(response.text)
 
 
 
 
 
 
 
 
 
1221
 
1222
+ answer = clean_answer(parsed.answer)
1223
  if not answer:
1224
+ raise RuntimeError("Gemini returned an empty answer.")
 
 
 
1225
  return answer
1226
 
1227
+ def adjudicate(self, question: str, evidence: str, first_answer: str, second_answer: str, media=None):
 
 
 
 
 
 
 
1228
  prompt = f"""
1229
  Two independent attempts produced different answers.
1230
 
 
1240
  CANDIDATE B:
1241
  {second_answer}
1242
 
1243
+ Do not choose by majority vote. Re-read the evidence and identify which
1244
+ candidate is directly supported. If neither is directly supported, derive a
1245
+ new answer from the evidence. Return only the exact requested answer.
 
 
 
1246
  """
1247
 
1248
+ contents = [prompt]
 
 
 
1249
  if media is not None:
1250
  contents.append(media)
1251
 
 
1253
  model=GEMINI_MODEL,
1254
  contents=contents,
1255
  config=types.GenerateContentConfig(
1256
+ system_instruction=self.system_instruction(),
 
1257
  max_output_tokens=2048,
1258
+ thinking_config=types.ThinkingConfig(thinking_level="high"),
1259
+ response_mime_type="application/json",
1260
+ response_schema=AnswerResponse,
 
 
 
 
 
1261
  ),
1262
  )
1263
 
1264
  if response.parsed:
1265
+ if isinstance(response.parsed, AnswerResponse):
 
 
 
1266
  parsed = response.parsed
1267
  else:
1268
+ parsed = AnswerResponse.model_validate(response.parsed)
 
 
 
1269
  else:
1270
+ parsed = AnswerResponse.model_validate_json(response.text)
 
 
1271
 
1272
+ return clean_answer(parsed.answer)
 
 
1273
 
1274
+ def solve_with_consensus(self, question: str, evidence: str, media=None):
 
 
 
 
 
1275
  first = self.generate_answer(
1276
  question,
1277
  evidence,
1278
  media,
1279
  independent=False,
1280
  )
1281
+ print(f"First model answer: {first}")
 
 
 
1282
 
1283
  time.sleep(1.0)
1284
 
 
1288
  media,
1289
  independent=True,
1290
  )
1291
+ print(f"Independent model answer: {second}")
1292
 
1293
+ if normalized_answer(first) == normalized_answer(second):
1294
+ print("Both model attempts agree.")
 
 
 
 
 
 
 
 
 
 
1295
  return first
1296
 
1297
+ print("Model attempts disagree. Running evidence-based adjudication.")
 
 
 
 
1298
  time.sleep(1.0)
1299
 
1300
  final = self.adjudicate(
 
1304
  second,
1305
  media,
1306
  )
1307
+ print(f"Adjudicated answer: {final}")
 
 
 
 
1308
  return final
1309
 
1310
+ def solve_task(self, task: dict):
 
 
 
1311
  task_id = task["task_id"]
1312
  question = task["question"]
1313
  file_name = task["file_name"]
1314
 
1315
  print("")
1316
+ print(f"Solving task {task_id}")
1317
+
1318
+ for solver in [
1319
+ self.solve_reversed_text,
1320
+ self.solve_operation_table,
1321
+ self.solve_libretexts_equine_vet,
1322
+ self.solve_1928_olympics,
1323
+ self.solve_pitchers_around_number,
1324
+ ]:
1325
+ answer = solver(question)
1326
+ if answer is not None:
1327
+ return answer
 
 
 
 
 
 
 
 
 
1328
 
1329
  path = None
 
1330
  if file_name:
1331
+ path = self.download_attachment(task_id, file_name)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1332
 
1333
+ if path is not None and path.suffix.lower() == ".py":
1334
+ answer = self.solve_python_file(path, question)
1335
+ if answer is not None:
1336
+ return answer
1337
 
1338
+ if path is not None and path.suffix.lower() in {".xlsx", ".xlsm"}:
1339
+ answer = self.solve_excel_food_sales(path, question)
1340
+ if answer is not None:
1341
+ return answer
 
 
 
 
 
 
 
1342
 
1343
+ if path is not None and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"}:
1344
+ answer = self.solve_chess_image(path, question)
1345
+ if answer is not None:
1346
+ return answer
 
 
 
 
1347
 
1348
+ youtube_transcript = self.fetch_youtube_transcript(question)
1349
  if youtube_transcript:
1350
  evidence = (
1351
+ "The following is the transcript of the YouTube video referenced "
1352
+ "in the question:\n\n" + youtube_transcript
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1353
  )
1354
+ return self.solve_with_consensus(question, evidence)
1355
 
1356
+ if path is not None and path.suffix.lower() in MEDIA_EXTENSIONS:
1357
+ print(f"Uploading media: {file_name}")
1358
+ media = self.client.files.upload(file=str(path))
1359
+ print("Media upload completed.")
1360
 
1361
  evidence = (
1362
+ "The actual media attachment referenced by the question is included "
1363
+ "in this request. Inspect the attachment itself carefully. "
 
1364
  f"Filename: {file_name}"
1365
  )
1366
+ return self.solve_with_consensus(question, evidence, media)
1367
 
1368
+ if path is not None and path.suffix.lower() in {".xlsx", ".xlsm"}:
1369
+ workbook_text = self.workbook_to_text(path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1370
  evidence = (
1371
+ "The following data was extracted from the attached workbook:\n\n"
 
1372
  + workbook_text
1373
  )
1374
+ return self.solve_with_consensus(question, evidence)
1375
 
1376
+ if path is not None and path.suffix.lower() in {".txt", ".csv", ".json", ".md", ".py"}:
1377
+ file_text = path.read_text(encoding="utf-8", errors="replace")[:70000]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1378
  evidence = (
1379
+ "The following is the content of the attached file:\n\n"
 
1380
  + file_text
1381
  )
1382
+ return self.solve_with_consensus(question, evidence)
1383
 
1384
+ print("Running targeted web research.")
1385
+ research = self.web_research(question)
1386
+ print(f"Web research collected: {len(research)} chars")
1387
+ return self.solve_with_consensus(question, research)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1388
 
1389
 
1390
+ def build_results_table(tasks: list[dict], cache: dict):
 
 
 
1391
  rows = []
1392
 
1393
  for task in tasks:
1394
+ task_id = task["task_id"]
 
 
 
1395
  rows.append(
1396
  {
1397
+ "Task ID": task_id,
1398
+ "Question": task["question"],
1399
+ "File": task["file_name"],
1400
+ "Answer": cache.get(task_id, ""),
1401
+ "Status": "READY" if task_id in cache else "PENDING",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1402
  }
1403
  )
1404
 
1405
+ return pd.DataFrame(rows)
 
 
1406
 
1407
 
1408
+ def run_and_submit_all(profile: gr.OAuthProfile | None):
 
 
1409
  if not profile:
1410
+ return "Please Login to Hugging Face.", None
 
 
 
 
 
 
 
1411
 
1412
+ username = str(profile.username).strip()
1413
+ space_id = os.getenv("SPACE_ID")
 
1414
 
1415
  if not space_id:
1416
+ return "SPACE_ID environment variable is missing.", None
 
 
 
 
 
 
 
1417
 
1418
+ questions_url = f"{DEFAULT_API_URL}/questions"
1419
+ submit_url = f"{DEFAULT_API_URL}/submit"
 
1420
 
1421
  try:
1422
  response = requests.get(
1423
  questions_url,
1424
+ headers={"User-Agent": USER_AGENT},
1425
  timeout=30,
1426
  )
 
1427
  response.raise_for_status()
1428
+ questions_data = response.json()
 
 
 
 
1429
  except Exception as error:
1430
+ return f"Could not fetch questions: {error}", None
 
 
 
1431
 
1432
  if not questions_data:
1433
+ return "No evaluation questions were returned.", None
 
 
 
1434
 
1435
  tasks = []
 
1436
  for item in questions_data:
1437
+ task_id = item.get("task_id")
1438
+ question = item.get("question")
1439
+ file_name = item.get("file_name") or ""
 
 
 
 
 
 
 
 
 
 
 
1440
 
1441
+ if not task_id or question is None:
1442
+ return "The questions API returned an invalid task.", None
 
 
 
 
 
 
 
1443
 
1444
  tasks.append(
1445
  {
1446
+ "task_id": task_id,
1447
+ "question": question,
1448
+ "file_name": file_name,
 
 
 
1449
  }
1450
  )
1451
 
1452
  try:
1453
  agent = BasicAgent()
 
1454
  except Exception as error:
1455
+ return f"Could not initialize agent: {error}", None
 
 
 
1456
 
1457
  cache = load_cache()
1458
+ valid_ids = {task["task_id"] for task in tasks}
 
 
 
 
 
1459
  cache = {
1460
  task_id: answer
1461
+ for task_id, answer in cache.items()
 
1462
  if task_id in valid_ids
1463
  }
 
1464
  save_cache(cache)
1465
 
1466
+ print(f"Cache file: {CACHE_FILE}")
1467
+ print(f"Total tasks: {len(tasks)}")
1468
+ print(f"Already solved: {len(cache)}")
1469
+ print(f"Remaining: {len(tasks) - len(cache)}")
 
 
 
 
 
 
 
1470
 
1471
  try:
1472
+ for index, task in enumerate(tasks, start=1):
1473
+ task_id = task["task_id"]
 
 
 
 
 
1474
 
1475
  if task_id in cache:
1476
  print(
1477
+ f"Skipping cached task {index}/{len(tasks)}: {task_id}"
 
 
1478
  )
1479
  continue
1480
 
1481
  print("")
1482
+ print(f"Processing task {index}/{len(tasks)}")
1483
+ answer = clean_answer(agent.solve_task(task))
 
 
 
 
 
 
 
 
 
 
1484
 
1485
  if not answer:
1486
+ raise RuntimeError(f"Task {task_id} produced an empty answer.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1487
 
1488
+ cache[task_id] = answer
1489
+ save_cache(cache)
1490
+ print(f"Saved answer for {task_id}: {answer}")
1491
  time.sleep(1.0)
1492
 
1493
  except Exception as error:
1494
  print(
1495
  "Evaluation paused: "
1496
+ f"{type(error).__name__}: {error}"
 
1497
  )
 
1498
  return (
1499
+ "Evaluation paused. Nothing was submitted.\n\n"
1500
+ f"Saved answers: {len(cache)}/{len(tasks)}\n"
1501
+ f"Remaining: {len(tasks) - len(cache)}\n\n"
1502
+ f"Error: {type(error).__name__}: {error}\n\n"
1503
+ "Click the same button again to resume from the saved answers.",
1504
+ build_results_table(tasks, cache),
 
 
 
 
 
 
 
 
 
 
 
1505
  )
1506
 
1507
  missing = [
1508
  task["task_id"]
1509
  for task in tasks
1510
+ if task["task_id"] not in cache
 
1511
  ]
1512
 
1513
  if missing:
1514
  return (
1515
+ "Evaluation is incomplete. Nothing was submitted.\n\n"
1516
+ f"Saved answers: {len(cache)}/{len(tasks)}",
1517
+ build_results_table(tasks, cache),
 
 
 
 
 
 
 
1518
  )
1519
 
1520
  answers_payload = [
1521
  {
1522
+ "task_id": task["task_id"],
1523
+ "submitted_answer": cache[task["task_id"]],
 
 
 
 
1524
  }
1525
  for task in tasks
1526
  ]
1527
 
1528
+ agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
 
 
 
 
1529
  submission_data = {
1530
+ "username": username,
1531
+ "agent_code": agent_code,
1532
+ "answers": answers_payload,
 
 
 
1533
  }
1534
 
1535
  print("")
1536
+ print("All 20 answers are ready.")
1537
+ print("Submitting answers...")
 
 
 
 
 
1538
 
1539
  try:
1540
  response = requests.post(
1541
  submit_url,
1542
  json=submission_data,
1543
+ headers={"User-Agent": USER_AGENT},
1544
  timeout=120,
1545
  )
 
1546
  response.raise_for_status()
1547
+ result = response.json()
 
 
 
 
1548
  except Exception as error:
1549
+ print(f"Submission failed: {error}")
 
 
 
1550
  return (
1551
+ "All 20 answers are saved, but submission failed.\n\n"
1552
+ f"Error: {error}\n\n"
1553
+ "Click the same button again. Saved answers will not be recomputed.",
1554
+ build_results_table(tasks, cache),
 
 
 
 
 
 
 
 
1555
  )
1556
 
1557
  final_status = (
1558
  "Submission Successful!\n"
1559
+ f"User: {result.get('username', username)}\n"
1560
+ f"Overall Score: {result.get('score', 'N/A')}% "
1561
+ f"({result.get('correct_count', '?')}/"
1562
+ f"{result.get('total_attempted', '?')} correct)\n"
1563
+ f"Message: {result.get('message', '')}"
 
 
 
 
 
 
 
 
 
1564
  )
1565
 
1566
+ print(final_status)
1567
+ return final_status, build_results_table(tasks, cache)
 
 
 
 
 
1568
 
1569
 
1570
  with gr.Blocks() as demo:
1571
+ gr.Markdown("# Basic Agent Evaluation Runner")
 
 
 
1572
  gr.Markdown(
1573
  """
1574
  **Instructions:**
1575
 
1576
  1. Log in to your Hugging Face account.
1577
  2. Click **Run / Resume Evaluation & Submit All Answers**.
1578
+ 3. Deterministic tasks are solved locally or from direct primary data where possible.
1579
  4. Research and media tasks are independently checked before acceptance.
1580
+ 5. Completed answers are cached when writable persistent storage is available, otherwise for the current Space runtime.
1581
  6. Submission occurs only after all 20 answers are ready.
1582
  """
1583
  )
 
1589
  )
1590
 
1591
  status_output = gr.Textbox(
1592
+ label="Run Status / Submission Result",
 
 
1593
  lines=10,
1594
  interactive=False,
1595
  )
1596
 
1597
  results_table = gr.DataFrame(
1598
+ label="Questions and Agent Answers",
 
 
1599
  wrap=True,
1600
  )
1601
 
1602
  run_button.click(
1603
  fn=run_and_submit_all,
1604
+ outputs=[status_output, results_table],
 
 
 
1605
  )
1606
 
1607
 
1608
  if __name__ == "__main__":
1609
+ print("\n" + "-" * 30 + " App Starting " + "-" * 30)
 
 
 
 
 
1610
 
1611
+ space_host = os.getenv("SPACE_HOST")
1612
+ space_id = os.getenv("SPACE_ID")
 
 
 
 
 
1613
 
1614
  if space_host:
1615
+ print(f"SPACE_HOST found: {space_host}")
 
 
 
1616
 
1617
  if space_id:
1618
+ print(f"SPACE_ID found: {space_id}")
 
 
 
 
1619
  print(
1620
  "Repo Tree URL: "
1621
+ f"https://huggingface.co/spaces/{space_id}/tree/main"
 
1622
  )
1623
 
1624
+ print("-" * 74)
1625
+ print("Launching Gradio Interface...")
 
 
 
 
 
1626
 
1627
  demo.launch(
1628
  debug=True,
1629
  share=False,
1630
  ssr_mode=False,
1631
+ )