Update app.py
Browse files
app.py
CHANGED
|
@@ -4,13 +4,16 @@ import re
|
|
| 4 |
import subprocess
|
| 5 |
import sys
|
| 6 |
import time
|
|
|
|
| 7 |
from pathlib import Path
|
| 8 |
from urllib.parse import parse_qs, urlparse
|
| 9 |
|
|
|
|
| 10 |
import gradio as gr
|
| 11 |
import pandas as pd
|
| 12 |
import requests
|
| 13 |
import spaces
|
|
|
|
| 14 |
from ddgs import DDGS
|
| 15 |
from google import genai
|
| 16 |
from google.genai import types
|
|
@@ -26,16 +29,20 @@ def _zerogpu_probe():
|
|
| 26 |
|
| 27 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 28 |
GEMINI_MODEL = "gemini-3.5-flash-lite"
|
|
|
|
| 29 |
|
| 30 |
GAIA_ATTACHMENT_FALLBACK_BASE = (
|
| 31 |
"https://huggingface.co/spaces/bstraehle/gaia/"
|
| 32 |
"resolve/809adb7a8f1747286eac400b8675dffdd973fbb9/data"
|
| 33 |
)
|
| 34 |
|
| 35 |
-
ATTACHMENT_DIR = Path("/tmp/
|
| 36 |
ATTACHMENT_DIR.mkdir(parents=True, exist_ok=True)
|
| 37 |
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
MEDIA_EXTENSIONS = {
|
| 41 |
".png",
|
|
@@ -49,6 +56,16 @@ MEDIA_EXTENSIONS = {
|
|
| 49 |
".ogg",
|
| 50 |
}
|
| 51 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
class AnswerResponse(BaseModel):
|
| 54 |
answer: str
|
|
@@ -58,18 +75,23 @@ class SearchPlan(BaseModel):
|
|
| 58 |
queries: list[str]
|
| 59 |
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
def load_cache() -> dict:
|
| 62 |
if not CACHE_FILE.exists():
|
| 63 |
return {}
|
| 64 |
|
| 65 |
try:
|
| 66 |
-
data = json.loads(
|
| 67 |
-
CACHE_FILE.read_text(encoding="utf-8")
|
| 68 |
-
)
|
| 69 |
-
|
| 70 |
if isinstance(data, dict):
|
| 71 |
return data
|
| 72 |
-
|
| 73 |
except Exception as error:
|
| 74 |
print(f"Could not load cache: {error}")
|
| 75 |
|
|
@@ -78,11 +100,7 @@ def load_cache() -> dict:
|
|
| 78 |
|
| 79 |
def save_cache(cache: dict):
|
| 80 |
CACHE_FILE.write_text(
|
| 81 |
-
json.dumps(
|
| 82 |
-
cache,
|
| 83 |
-
ensure_ascii=False,
|
| 84 |
-
indent=2,
|
| 85 |
-
),
|
| 86 |
encoding="utf-8",
|
| 87 |
)
|
| 88 |
|
|
@@ -110,72 +128,55 @@ def clean_answer(answer: str) -> str:
|
|
| 110 |
|
| 111 |
if answer.startswith("```") and answer.endswith("```"):
|
| 112 |
answer = answer[3:-3].strip()
|
| 113 |
-
|
| 114 |
first_line, separator, remainder = answer.partition("\n")
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
"plaintext",
|
| 122 |
-
"python",
|
| 123 |
-
"json",
|
| 124 |
-
}
|
| 125 |
-
):
|
| 126 |
answer = remainder.strip()
|
| 127 |
|
| 128 |
return answer.strip("`").strip()
|
| 129 |
|
| 130 |
|
| 131 |
def normalized_answer(answer: str) -> str:
|
| 132 |
-
return re.sub(
|
| 133 |
-
r"\s+",
|
| 134 |
-
" ",
|
| 135 |
-
clean_answer(answer),
|
| 136 |
-
).strip().casefold()
|
| 137 |
|
| 138 |
|
| 139 |
def extract_youtube_id(text: str):
|
| 140 |
-
urls = re.findall(
|
| 141 |
-
r"https?://[^\s)>]+",
|
| 142 |
-
text,
|
| 143 |
-
)
|
| 144 |
|
| 145 |
for url in urls:
|
| 146 |
url = url.rstrip(".,;!?")
|
| 147 |
-
|
| 148 |
try:
|
| 149 |
parsed = urlparse(url)
|
| 150 |
-
|
| 151 |
if "youtube.com" in parsed.netloc:
|
| 152 |
-
video_id = parse_qs(
|
| 153 |
-
parsed.query
|
| 154 |
-
).get("v")
|
| 155 |
-
|
| 156 |
if video_id:
|
| 157 |
return video_id[0]
|
| 158 |
-
|
| 159 |
if "youtu.be" in parsed.netloc:
|
| 160 |
value = parsed.path.strip("/")
|
| 161 |
-
|
| 162 |
if value:
|
| 163 |
return value
|
| 164 |
-
|
| 165 |
except Exception:
|
| 166 |
continue
|
| 167 |
|
| 168 |
return None
|
| 169 |
|
| 170 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
class BasicAgent:
|
| 172 |
def __init__(self):
|
| 173 |
api_key = os.getenv("GEMINI_API_KEY")
|
| 174 |
-
|
| 175 |
if not api_key:
|
| 176 |
-
raise RuntimeError(
|
| 177 |
-
"GEMINI_API_KEY secret is not configured."
|
| 178 |
-
)
|
| 179 |
|
| 180 |
self.client = genai.Client(
|
| 181 |
api_key=api_key,
|
|
@@ -186,143 +187,71 @@ class BasicAgent:
|
|
| 186 |
exp_base=2.0,
|
| 187 |
max_delay=30.0,
|
| 188 |
jitter=1.0,
|
| 189 |
-
http_status_codes=[
|
| 190 |
-
408,
|
| 191 |
-
500,
|
| 192 |
-
502,
|
| 193 |
-
503,
|
| 194 |
-
504,
|
| 195 |
-
],
|
| 196 |
)
|
| 197 |
),
|
| 198 |
)
|
| 199 |
|
| 200 |
-
print(
|
| 201 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 202 |
)
|
| 203 |
|
| 204 |
-
def download_attachment(
|
| 205 |
-
self,
|
| 206 |
-
task_id: str,
|
| 207 |
-
file_name: str,
|
| 208 |
-
) -> Path:
|
| 209 |
safe_name = os.path.basename(file_name)
|
| 210 |
-
|
| 211 |
if not safe_name:
|
| 212 |
-
raise RuntimeError(
|
| 213 |
-
"Attachment file name is empty."
|
| 214 |
-
)
|
| 215 |
-
|
| 216 |
-
destination = (
|
| 217 |
-
ATTACHMENT_DIR
|
| 218 |
-
/ f"{task_id}_{safe_name}"
|
| 219 |
-
)
|
| 220 |
-
|
| 221 |
-
if (
|
| 222 |
-
destination.exists()
|
| 223 |
-
and destination.stat().st_size > 0
|
| 224 |
-
):
|
| 225 |
-
print(
|
| 226 |
-
f"Using cached attachment: {safe_name}"
|
| 227 |
-
)
|
| 228 |
|
|
|
|
|
|
|
|
|
|
| 229 |
return destination
|
| 230 |
|
| 231 |
-
official_url =
|
| 232 |
-
f"{DEFAULT_API_URL}/files/{task_id}"
|
| 233 |
-
)
|
| 234 |
-
|
| 235 |
official_error = None
|
| 236 |
|
| 237 |
try:
|
| 238 |
-
print(
|
| 239 |
-
|
| 240 |
-
f"{official_url}"
|
| 241 |
-
)
|
| 242 |
-
|
| 243 |
-
response = requests.get(
|
| 244 |
-
official_url,
|
| 245 |
-
timeout=30,
|
| 246 |
-
)
|
| 247 |
-
|
| 248 |
response.raise_for_status()
|
| 249 |
-
|
| 250 |
if not response.content:
|
| 251 |
-
raise RuntimeError(
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
destination.write_bytes(
|
| 256 |
-
response.content
|
| 257 |
-
)
|
| 258 |
-
|
| 259 |
-
print(
|
| 260 |
-
"Downloaded attachment from official endpoint: "
|
| 261 |
-
f"{safe_name}"
|
| 262 |
-
)
|
| 263 |
-
|
| 264 |
return destination
|
| 265 |
-
|
| 266 |
except Exception as error:
|
| 267 |
official_error = error
|
| 268 |
-
|
| 269 |
print(
|
| 270 |
"Official attachment download failed: "
|
| 271 |
f"{type(error).__name__}: {error}"
|
| 272 |
)
|
| 273 |
|
| 274 |
fallback_url = (
|
| 275 |
-
f"{GAIA_ATTACHMENT_FALLBACK_BASE}/"
|
| 276 |
-
f"{safe_name}?download=true"
|
| 277 |
-
)
|
| 278 |
-
|
| 279 |
-
print(
|
| 280 |
-
f"Trying attachment fallback: {fallback_url}"
|
| 281 |
)
|
|
|
|
| 282 |
|
| 283 |
try:
|
| 284 |
-
response =
|
| 285 |
-
fallback_url,
|
| 286 |
-
timeout=60,
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
response.raise_for_status()
|
| 290 |
-
|
| 291 |
if not response.content:
|
| 292 |
-
raise RuntimeError(
|
| 293 |
-
"Fallback returned an empty file."
|
| 294 |
-
)
|
| 295 |
-
|
| 296 |
-
content_type = (
|
| 297 |
-
response.headers
|
| 298 |
-
.get("content-type", "")
|
| 299 |
-
.lower()
|
| 300 |
-
)
|
| 301 |
-
|
| 302 |
-
suffix = Path(
|
| 303 |
-
safe_name
|
| 304 |
-
).suffix.lower()
|
| 305 |
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
):
|
| 310 |
raise RuntimeError(
|
| 311 |
-
"Fallback returned HTML instead "
|
| 312 |
-
"of the requested attachment."
|
| 313 |
)
|
| 314 |
|
| 315 |
-
destination.write_bytes(
|
| 316 |
-
|
| 317 |
-
)
|
| 318 |
-
|
| 319 |
-
print(
|
| 320 |
-
"Downloaded attachment from fallback: "
|
| 321 |
-
f"{safe_name}"
|
| 322 |
-
)
|
| 323 |
-
|
| 324 |
return destination
|
| 325 |
-
|
| 326 |
except Exception as fallback_error:
|
| 327 |
raise RuntimeError(
|
| 328 |
"Unable to download attachment. "
|
|
@@ -330,40 +259,21 @@ class BasicAgent:
|
|
| 330 |
f"Fallback error: {fallback_error}"
|
| 331 |
)
|
| 332 |
|
| 333 |
-
def solve_reversed_text(
|
| 334 |
-
self,
|
| 335 |
-
question: str,
|
| 336 |
-
):
|
| 337 |
reversed_question = question[::-1]
|
| 338 |
-
|
| 339 |
lowered = reversed_question.lower()
|
| 340 |
|
| 341 |
-
if
|
| 342 |
-
"opposite of the word" not in lowered
|
| 343 |
-
and "opposite of" not in lowered
|
| 344 |
-
):
|
| 345 |
return None
|
| 346 |
|
| 347 |
-
|
|
|
|
| 348 |
r'opposite of the word\s+["\']([^"\']+)["\']',
|
| 349 |
r'opposite of\s+["\']([^"\']+)["\']',
|
| 350 |
-
]
|
| 351 |
-
|
| 352 |
-
target = None
|
| 353 |
-
|
| 354 |
-
for pattern in patterns:
|
| 355 |
-
match = re.search(
|
| 356 |
-
pattern,
|
| 357 |
-
reversed_question,
|
| 358 |
-
flags=re.IGNORECASE,
|
| 359 |
-
)
|
| 360 |
-
|
| 361 |
if match:
|
| 362 |
-
target = (
|
| 363 |
-
match.group(1)
|
| 364 |
-
.strip()
|
| 365 |
-
.lower()
|
| 366 |
-
)
|
| 367 |
break
|
| 368 |
|
| 369 |
if not target:
|
|
@@ -385,24 +295,13 @@ class BasicAgent:
|
|
| 385 |
}
|
| 386 |
|
| 387 |
answer = opposites.get(target)
|
| 388 |
-
|
| 389 |
if answer:
|
| 390 |
-
print(
|
| 391 |
-
f"Local reversed-text solver: {answer}"
|
| 392 |
-
)
|
| 393 |
-
|
| 394 |
return answer
|
| 395 |
|
| 396 |
-
def solve_operation_table(
|
| 397 |
-
self,
|
| 398 |
-
question: str,
|
| 399 |
-
):
|
| 400 |
lowered = question.lower()
|
| 401 |
-
|
| 402 |
-
if (
|
| 403 |
-
"commutative" not in lowered
|
| 404 |
-
or "|" not in question
|
| 405 |
-
):
|
| 406 |
return None
|
| 407 |
|
| 408 |
set_match = re.search(
|
|
@@ -410,7 +309,6 @@ class BasicAgent:
|
|
| 410 |
question,
|
| 411 |
flags=re.IGNORECASE,
|
| 412 |
)
|
| 413 |
-
|
| 414 |
if not set_match:
|
| 415 |
return None
|
| 416 |
|
|
@@ -419,167 +317,83 @@ class BasicAgent:
|
|
| 419 |
for value in set_match.group(1).split(",")
|
| 420 |
if value.strip()
|
| 421 |
]
|
| 422 |
-
|
| 423 |
if len(elements) < 2:
|
| 424 |
return None
|
| 425 |
|
| 426 |
-
|
| 427 |
-
r"\|([^|]*)",
|
| 428 |
-
question,
|
| 429 |
-
)
|
| 430 |
-
|
| 431 |
-
tokens = [
|
| 432 |
-
token.strip()
|
| 433 |
-
for token in raw_tokens
|
| 434 |
-
]
|
| 435 |
-
|
| 436 |
rows = {}
|
| 437 |
n = len(elements)
|
| 438 |
-
|
| 439 |
index = 0
|
| 440 |
|
| 441 |
while index < len(tokens):
|
| 442 |
-
if (
|
| 443 |
-
tokens[index
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
candidate = tokens[
|
| 447 |
-
index + 1:
|
| 448 |
-
index + 1 + n
|
| 449 |
-
]
|
| 450 |
-
|
| 451 |
-
if (
|
| 452 |
-
len(candidate) == n
|
| 453 |
-
and all(
|
| 454 |
-
value in elements
|
| 455 |
-
for value in candidate
|
| 456 |
-
)
|
| 457 |
-
):
|
| 458 |
-
rows[
|
| 459 |
-
tokens[index]
|
| 460 |
-
] = dict(
|
| 461 |
-
zip(
|
| 462 |
-
elements,
|
| 463 |
-
candidate,
|
| 464 |
-
)
|
| 465 |
-
)
|
| 466 |
-
|
| 467 |
index += n + 1
|
| 468 |
continue
|
| 469 |
-
|
| 470 |
index += 1
|
| 471 |
|
| 472 |
if len(rows) != n:
|
| 473 |
return None
|
| 474 |
|
| 475 |
involved = set()
|
| 476 |
-
|
| 477 |
for left in elements:
|
| 478 |
for right in elements:
|
| 479 |
-
if (
|
| 480 |
-
left in rows
|
| 481 |
-
and right in rows
|
| 482 |
-
and rows[left].get(right)
|
| 483 |
-
!= rows[right].get(left)
|
| 484 |
-
):
|
| 485 |
involved.add(left)
|
| 486 |
involved.add(right)
|
| 487 |
|
| 488 |
if not involved:
|
| 489 |
return None
|
| 490 |
|
| 491 |
-
answer = ", ".join(
|
| 492 |
-
|
| 493 |
-
)
|
| 494 |
-
|
| 495 |
-
print(
|
| 496 |
-
f"Local operation-table solver: {answer}"
|
| 497 |
-
)
|
| 498 |
-
|
| 499 |
return answer
|
| 500 |
|
| 501 |
-
def solve_python_file(
|
| 502 |
-
|
| 503 |
-
path: Path,
|
| 504 |
-
question: str,
|
| 505 |
-
):
|
| 506 |
-
if (
|
| 507 |
-
path.suffix.lower() != ".py"
|
| 508 |
-
or "numeric output"
|
| 509 |
-
not in question.lower()
|
| 510 |
-
):
|
| 511 |
return None
|
| 512 |
|
| 513 |
try:
|
| 514 |
completed = subprocess.run(
|
| 515 |
-
[
|
| 516 |
-
sys.executable,
|
| 517 |
-
"-I",
|
| 518 |
-
str(path),
|
| 519 |
-
],
|
| 520 |
cwd=str(path.parent),
|
| 521 |
capture_output=True,
|
| 522 |
text=True,
|
| 523 |
timeout=15,
|
| 524 |
)
|
| 525 |
-
|
| 526 |
except Exception as error:
|
| 527 |
-
print(
|
| 528 |
-
f"Local Python execution failed: {error}"
|
| 529 |
-
)
|
| 530 |
return None
|
| 531 |
|
| 532 |
stdout = completed.stdout.strip()
|
| 533 |
-
|
| 534 |
if not stdout:
|
| 535 |
-
print(
|
| 536 |
-
"Python script produced no stdout."
|
| 537 |
-
)
|
| 538 |
return None
|
| 539 |
|
| 540 |
-
lines = [
|
| 541 |
-
line.strip()
|
| 542 |
-
for line in stdout.splitlines()
|
| 543 |
-
if line.strip()
|
| 544 |
-
]
|
| 545 |
-
|
| 546 |
if not lines:
|
| 547 |
return None
|
| 548 |
|
| 549 |
-
last_line = lines[-1]
|
| 550 |
-
|
| 551 |
numeric_matches = re.findall(
|
| 552 |
r"-?(?:\d+(?:\.\d+)?|\.\d+)",
|
| 553 |
-
|
| 554 |
)
|
| 555 |
-
|
| 556 |
if not numeric_matches:
|
| 557 |
return None
|
| 558 |
|
| 559 |
answer = numeric_matches[-1]
|
| 560 |
-
|
| 561 |
-
print(
|
| 562 |
-
f"Local Python solver: {answer}"
|
| 563 |
-
)
|
| 564 |
-
|
| 565 |
return answer
|
| 566 |
|
| 567 |
-
def solve_excel_food_sales(
|
| 568 |
-
self,
|
| 569 |
-
path: Path,
|
| 570 |
-
question: str,
|
| 571 |
-
):
|
| 572 |
lowered = question.lower()
|
| 573 |
-
|
| 574 |
if (
|
| 575 |
-
path.suffix.lower()
|
| 576 |
-
not in {".xlsx", ".xlsm"}
|
| 577 |
or "sales" not in lowered
|
| 578 |
or "food" not in lowered
|
| 579 |
-
or (
|
| 580 |
-
"drink" not in lowered
|
| 581 |
-
and "beverage" not in lowered
|
| 582 |
-
)
|
| 583 |
):
|
| 584 |
return None
|
| 585 |
|
|
@@ -621,248 +435,624 @@ class BasicAgent:
|
|
| 621 |
|
| 622 |
try:
|
| 623 |
workbook = pd.ExcelFile(path)
|
| 624 |
-
|
| 625 |
grand_total = 0.0
|
| 626 |
found_food_columns = 0
|
| 627 |
found_drink_columns = 0
|
| 628 |
|
| 629 |
for sheet in workbook.sheet_names:
|
| 630 |
-
frame = pd.read_excel(
|
| 631 |
-
path,
|
| 632 |
-
sheet_name=sheet,
|
| 633 |
-
)
|
| 634 |
-
|
| 635 |
if frame.empty:
|
| 636 |
continue
|
| 637 |
|
| 638 |
-
numeric_columns = []
|
| 639 |
-
|
| 640 |
for column in frame.columns:
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
errors="coerce",
|
| 644 |
-
)
|
| 645 |
-
|
| 646 |
-
if converted.notna().any():
|
| 647 |
-
numeric_columns.append(
|
| 648 |
-
(
|
| 649 |
-
column,
|
| 650 |
-
converted,
|
| 651 |
-
)
|
| 652 |
-
)
|
| 653 |
-
|
| 654 |
-
for column, values in numeric_columns:
|
| 655 |
-
column_name = str(
|
| 656 |
-
column
|
| 657 |
-
).strip().lower()
|
| 658 |
-
|
| 659 |
-
if any(
|
| 660 |
-
word in column_name
|
| 661 |
-
for word in metadata_words
|
| 662 |
-
):
|
| 663 |
continue
|
| 664 |
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
):
|
| 669 |
found_drink_columns += 1
|
| 670 |
continue
|
| 671 |
|
| 672 |
-
grand_total += float(
|
| 673 |
-
values.fillna(0).sum()
|
| 674 |
-
)
|
| 675 |
-
|
| 676 |
found_food_columns += 1
|
| 677 |
|
| 678 |
-
if
|
| 679 |
-
found_food_columns == 0
|
| 680 |
-
or found_drink_columns == 0
|
| 681 |
-
):
|
| 682 |
return None
|
| 683 |
|
| 684 |
answer = f"{grand_total:.2f}"
|
| 685 |
-
|
| 686 |
print(
|
| 687 |
-
"Local Excel solver:"
|
| 688 |
-
f"
|
| 689 |
-
f"
|
| 690 |
-
f"
|
| 691 |
)
|
| 692 |
-
|
| 693 |
return answer
|
| 694 |
-
|
| 695 |
except Exception as error:
|
| 696 |
-
print(
|
| 697 |
-
|
| 698 |
-
)
|
| 699 |
|
|
|
|
|
|
|
|
|
|
| 700 |
return None
|
| 701 |
|
| 702 |
-
|
| 703 |
-
self,
|
| 704 |
-
path: Path,
|
| 705 |
-
) -> str:
|
| 706 |
-
workbook = load_workbook(
|
| 707 |
-
path,
|
| 708 |
-
data_only=True,
|
| 709 |
-
read_only=True,
|
| 710 |
-
)
|
| 711 |
|
| 712 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 713 |
|
| 714 |
try:
|
| 715 |
-
|
| 716 |
-
|
| 717 |
-
|
| 718 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 719 |
|
| 720 |
-
|
| 721 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 722 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 723 |
|
| 724 |
-
|
| 725 |
-
|
| 726 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 727 |
)
|
|
|
|
|
|
|
|
|
|
| 728 |
|
| 729 |
-
|
| 730 |
-
|
| 731 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 732 |
)
|
|
|
|
|
|
|
| 733 |
|
| 734 |
-
|
| 735 |
-
1,
|
| 736 |
-
max_rows + 1,
|
| 737 |
-
):
|
| 738 |
-
values = []
|
| 739 |
|
| 740 |
-
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
):
|
| 744 |
-
value = sheet.cell(
|
| 745 |
-
row=row_number,
|
| 746 |
-
column=column_number,
|
| 747 |
-
).value
|
| 748 |
|
| 749 |
-
|
| 750 |
-
|
| 751 |
-
else:
|
| 752 |
-
values.append(
|
| 753 |
-
str(value)
|
| 754 |
-
)
|
| 755 |
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
)
|
| 760 |
|
| 761 |
-
|
| 762 |
-
|
| 763 |
|
| 764 |
-
|
|
|
|
|
|
|
| 765 |
|
| 766 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 767 |
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
):
|
| 772 |
-
video_id = extract_youtube_id(
|
| 773 |
-
question
|
| 774 |
)
|
| 775 |
|
| 776 |
-
|
| 777 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 778 |
|
| 779 |
print(
|
| 780 |
-
|
|
|
|
| 781 |
)
|
| 782 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 783 |
try:
|
| 784 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 785 |
|
| 786 |
-
|
| 787 |
-
|
| 788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
)
|
| 790 |
|
| 791 |
-
|
| 792 |
-
|
| 793 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 794 |
)
|
| 795 |
|
| 796 |
-
if
|
| 797 |
-
print(
|
| 798 |
-
|
| 799 |
-
f"{len(text)} chars"
|
| 800 |
-
)
|
| 801 |
|
| 802 |
-
|
|
|
|
|
|
|
| 803 |
|
| 804 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 805 |
print(
|
| 806 |
-
"
|
| 807 |
-
f"{
|
| 808 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 809 |
|
| 810 |
try:
|
| 811 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 812 |
|
| 813 |
-
|
| 814 |
-
|
| 815 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 816 |
|
|
|
|
|
|
|
|
|
|
| 817 |
for transcript in transcript_list:
|
| 818 |
try:
|
| 819 |
if transcript.language_code == "en":
|
| 820 |
fetched = transcript.fetch()
|
| 821 |
-
|
| 822 |
elif transcript.is_translatable:
|
| 823 |
-
fetched = (
|
| 824 |
-
transcript
|
| 825 |
-
.translate("en")
|
| 826 |
-
.fetch()
|
| 827 |
-
)
|
| 828 |
-
|
| 829 |
else:
|
| 830 |
continue
|
| 831 |
|
| 832 |
-
text = " ".join(
|
| 833 |
-
snippet.text
|
| 834 |
-
for snippet in fetched
|
| 835 |
-
)
|
| 836 |
-
|
| 837 |
if text.strip():
|
| 838 |
print(
|
| 839 |
-
"Fallback YouTube transcript "
|
| 840 |
-
f"
|
| 841 |
)
|
| 842 |
-
|
| 843 |
return text[:60000]
|
| 844 |
-
|
| 845 |
except Exception:
|
| 846 |
continue
|
| 847 |
-
|
| 848 |
except Exception as error:
|
| 849 |
-
print(
|
| 850 |
-
f"YouTube transcript unavailable: {error}"
|
| 851 |
-
)
|
| 852 |
|
| 853 |
return ""
|
| 854 |
|
| 855 |
-
def plan_search_queries(
|
| 856 |
-
self,
|
| 857 |
-
question: str,
|
| 858 |
-
):
|
| 859 |
prompt = f"""
|
| 860 |
-
Create 3 concise web search queries that would help solve
|
| 861 |
-
|
| 862 |
-
|
| 863 |
-
Do not answer the question.
|
| 864 |
-
Do not search for the benchmark or task ID.
|
| 865 |
-
Search for the underlying real-world information.
|
| 866 |
|
| 867 |
QUESTION:
|
| 868 |
{question}
|
|
@@ -880,244 +1070,116 @@ QUESTION:
|
|
| 880 |
)
|
| 881 |
|
| 882 |
if response.parsed:
|
| 883 |
-
if isinstance(
|
| 884 |
-
response.parsed,
|
| 885 |
-
SearchPlan,
|
| 886 |
-
):
|
| 887 |
plan = response.parsed
|
| 888 |
else:
|
| 889 |
-
plan = SearchPlan.model_validate(
|
| 890 |
-
response.parsed
|
| 891 |
-
)
|
| 892 |
-
|
| 893 |
else:
|
| 894 |
-
plan = SearchPlan.model_validate_json(
|
| 895 |
-
response.text
|
| 896 |
-
)
|
| 897 |
-
|
| 898 |
-
queries = [
|
| 899 |
-
query.strip()
|
| 900 |
-
for query in plan.queries
|
| 901 |
-
if query.strip()
|
| 902 |
-
]
|
| 903 |
|
|
|
|
| 904 |
if queries:
|
| 905 |
-
print(
|
| 906 |
-
"Search queries: "
|
| 907 |
-
+ " | ".join(queries[:3])
|
| 908 |
-
)
|
| 909 |
-
|
| 910 |
return queries[:3]
|
| 911 |
-
|
| 912 |
except Exception as error:
|
| 913 |
-
print(
|
| 914 |
-
f"Search-query planning failed: {error}"
|
| 915 |
-
)
|
| 916 |
-
|
| 917 |
-
simplified = re.sub(
|
| 918 |
-
r"https?://\S+",
|
| 919 |
-
" ",
|
| 920 |
-
question,
|
| 921 |
-
)
|
| 922 |
-
|
| 923 |
-
simplified = re.sub(
|
| 924 |
-
r"\s+",
|
| 925 |
-
" ",
|
| 926 |
-
simplified,
|
| 927 |
-
).strip()
|
| 928 |
-
|
| 929 |
-
return [
|
| 930 |
-
simplified[:250]
|
| 931 |
-
]
|
| 932 |
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
|
| 936 |
-
) -> str:
|
| 937 |
-
queries = self.plan_search_queries(
|
| 938 |
-
question
|
| 939 |
-
)
|
| 940 |
|
|
|
|
|
|
|
| 941 |
results_output = []
|
| 942 |
urls = []
|
| 943 |
seen_urls = set()
|
| 944 |
|
| 945 |
-
for query_number, query in enumerate(
|
| 946 |
-
queries,
|
| 947 |
-
start=1,
|
| 948 |
-
):
|
| 949 |
try:
|
| 950 |
-
results = DDGS(
|
| 951 |
-
timeout=15,
|
| 952 |
-
).text(
|
| 953 |
query,
|
| 954 |
region="us-en",
|
| 955 |
safesearch="moderate",
|
| 956 |
max_results=8,
|
| 957 |
)
|
| 958 |
-
|
| 959 |
except Exception as error:
|
| 960 |
-
print(
|
| 961 |
-
f"Search failed for '{query}': {error}"
|
| 962 |
-
)
|
| 963 |
-
continue
|
| 964 |
-
|
| 965 |
-
if not results:
|
| 966 |
continue
|
| 967 |
|
| 968 |
-
for index, result in enumerate(
|
| 969 |
-
|
| 970 |
-
|
| 971 |
-
|
| 972 |
-
title = (
|
| 973 |
-
result.get("title")
|
| 974 |
-
or ""
|
| 975 |
-
)
|
| 976 |
-
|
| 977 |
-
url = (
|
| 978 |
-
result.get("href")
|
| 979 |
-
or result.get("url")
|
| 980 |
-
or ""
|
| 981 |
-
)
|
| 982 |
-
|
| 983 |
-
snippet = (
|
| 984 |
-
result.get("body")
|
| 985 |
-
or result.get("description")
|
| 986 |
-
or ""
|
| 987 |
-
)
|
| 988 |
|
| 989 |
results_output.append(
|
| 990 |
-
f"\nSEARCH QUERY {query_number}, "
|
| 991 |
-
f"RESULT {index}\n"
|
| 992 |
f"Title: {title}\n"
|
| 993 |
f"URL: {url}\n"
|
| 994 |
f"Snippet: {snippet}\n"
|
| 995 |
)
|
| 996 |
|
| 997 |
-
if (
|
| 998 |
-
url
|
| 999 |
-
and url not in seen_urls
|
| 1000 |
-
and len(urls) < 6
|
| 1001 |
-
):
|
| 1002 |
urls.append(url)
|
| 1003 |
seen_urls.add(url)
|
| 1004 |
|
| 1005 |
-
for index, url in enumerate(
|
| 1006 |
-
urls,
|
| 1007 |
-
start=1,
|
| 1008 |
-
):
|
| 1009 |
try:
|
| 1010 |
-
page = DDGS(
|
| 1011 |
-
|
| 1012 |
-
|
| 1013 |
-
url,
|
| 1014 |
-
fmt="text_plain",
|
| 1015 |
-
)
|
| 1016 |
-
|
| 1017 |
-
if isinstance(
|
| 1018 |
-
page,
|
| 1019 |
-
dict,
|
| 1020 |
-
):
|
| 1021 |
-
page_text = str(
|
| 1022 |
-
page.get(
|
| 1023 |
-
"content",
|
| 1024 |
-
"",
|
| 1025 |
-
)
|
| 1026 |
-
)
|
| 1027 |
-
|
| 1028 |
else:
|
| 1029 |
-
page_text = str(
|
| 1030 |
-
page or ""
|
| 1031 |
-
)
|
| 1032 |
|
| 1033 |
if not page_text.strip():
|
| 1034 |
continue
|
| 1035 |
|
| 1036 |
-
page_text = page_text[:10000]
|
| 1037 |
-
|
| 1038 |
results_output.append(
|
| 1039 |
-
f"\nWEB PAGE {index}\n"
|
| 1040 |
-
f"URL: {url}\n"
|
| 1041 |
-
f"{page_text}\n"
|
| 1042 |
)
|
| 1043 |
-
|
| 1044 |
except Exception as error:
|
| 1045 |
-
print(
|
| 1046 |
-
f"Could not read {url}: {error}"
|
| 1047 |
-
)
|
| 1048 |
-
|
| 1049 |
-
research = "\n".join(
|
| 1050 |
-
results_output
|
| 1051 |
-
)
|
| 1052 |
|
|
|
|
| 1053 |
if not research.strip():
|
| 1054 |
-
return
|
| 1055 |
-
"No useful web research could be retrieved."
|
| 1056 |
-
)
|
| 1057 |
|
| 1058 |
return research[:50000]
|
| 1059 |
|
| 1060 |
-
def system_instruction(
|
| 1061 |
-
self,
|
| 1062 |
-
):
|
| 1063 |
return """
|
| 1064 |
You are solving one GAIA Level 1 benchmark task.
|
| 1065 |
|
| 1066 |
Accuracy is more important than speed.
|
| 1067 |
-
|
| 1068 |
Use only the question and evidence supplied in this request.
|
|
|
|
|
|
|
| 1069 |
Do not assume the first search result is correct.
|
| 1070 |
Cross-check names, dates, numbers and relationships.
|
| 1071 |
|
| 1072 |
The evaluator uses exact-match scoring.
|
| 1073 |
-
|
| 1074 |
Return ONLY the requested final answer.
|
| 1075 |
-
|
| 1076 |
-
|
| 1077 |
-
|
| 1078 |
-
|
| 1079 |
-
|
| 1080 |
-
|
| 1081 |
-
|
| 1082 |
-
|
| 1083 |
-
|
| 1084 |
-
|
| 1085 |
-
|
| 1086 |
-
|
| 1087 |
-
|
| 1088 |
-
- Do not arbitrarily lowercase answers.
|
| 1089 |
-
- If the question asks for a city without abbreviations,
|
| 1090 |
-
spell the city name out fully.
|
| 1091 |
-
- If the question requests first name only, return first name only.
|
| 1092 |
-
- If the question requests surname only, return surname only.
|
| 1093 |
-
- If the question requests an IOC code, return only that code.
|
| 1094 |
-
- If the question requests a comma-separated list, use exactly
|
| 1095 |
-
that format and preserve the requested ordering.
|
| 1096 |
-
- If it asks for alphabetical order, alphabetize exactly.
|
| 1097 |
-
- For numeric answers, do not add units or symbols unless the
|
| 1098 |
-
requested output explicitly requires them.
|
| 1099 |
-
- If USD is requested with two decimal places but no explicit
|
| 1100 |
-
request for a currency symbol, return the number with exactly
|
| 1101 |
-
two decimal places.
|
| 1102 |
-
- Carefully distinguish similar people, titles and dates.
|
| 1103 |
"""
|
| 1104 |
|
| 1105 |
-
def generate_answer(
|
| 1106 |
-
|
| 1107 |
-
|
| 1108 |
-
|
| 1109 |
-
|
| 1110 |
-
|
| 1111 |
-
|
| 1112 |
-
if independent:
|
| 1113 |
-
instruction = """
|
| 1114 |
-
Solve the question independently from the evidence.
|
| 1115 |
-
Do not assume any previous candidate answer is correct.
|
| 1116 |
-
"""
|
| 1117 |
-
else:
|
| 1118 |
-
instruction = """
|
| 1119 |
-
Solve the question carefully from the evidence.
|
| 1120 |
-
"""
|
| 1121 |
|
| 1122 |
prompt = f"""
|
| 1123 |
{instruction}
|
|
@@ -1131,10 +1193,7 @@ EVIDENCE:
|
|
| 1131 |
Return only the exact requested answer.
|
| 1132 |
"""
|
| 1133 |
|
| 1134 |
-
contents = [
|
| 1135 |
-
prompt
|
| 1136 |
-
]
|
| 1137 |
-
|
| 1138 |
if media is not None:
|
| 1139 |
contents.append(media)
|
| 1140 |
|
|
@@ -1142,60 +1201,30 @@ Return only the exact requested answer.
|
|
| 1142 |
model=GEMINI_MODEL,
|
| 1143 |
contents=contents,
|
| 1144 |
config=types.GenerateContentConfig(
|
| 1145 |
-
system_instruction=
|
| 1146 |
-
self.system_instruction(),
|
| 1147 |
max_output_tokens=2048,
|
| 1148 |
-
thinking_config=
|
| 1149 |
-
|
| 1150 |
-
|
| 1151 |
-
),
|
| 1152 |
-
response_mime_type=
|
| 1153 |
-
"application/json",
|
| 1154 |
-
response_schema=
|
| 1155 |
-
AnswerResponse,
|
| 1156 |
),
|
| 1157 |
)
|
| 1158 |
|
| 1159 |
if response.parsed:
|
| 1160 |
-
if isinstance(
|
| 1161 |
-
response.parsed,
|
| 1162 |
-
AnswerResponse,
|
| 1163 |
-
):
|
| 1164 |
parsed = response.parsed
|
| 1165 |
else:
|
| 1166 |
-
parsed = AnswerResponse.model_validate(
|
| 1167 |
-
response.parsed
|
| 1168 |
-
)
|
| 1169 |
-
|
| 1170 |
else:
|
| 1171 |
if not response.text:
|
| 1172 |
-
raise RuntimeError(
|
| 1173 |
-
|
| 1174 |
-
)
|
| 1175 |
-
|
| 1176 |
-
parsed = AnswerResponse.model_validate_json(
|
| 1177 |
-
response.text
|
| 1178 |
-
)
|
| 1179 |
-
|
| 1180 |
-
answer = clean_answer(
|
| 1181 |
-
parsed.answer
|
| 1182 |
-
)
|
| 1183 |
|
|
|
|
| 1184 |
if not answer:
|
| 1185 |
-
raise RuntimeError(
|
| 1186 |
-
"Gemini returned an empty answer."
|
| 1187 |
-
)
|
| 1188 |
-
|
| 1189 |
return answer
|
| 1190 |
|
| 1191 |
-
def adjudicate(
|
| 1192 |
-
self,
|
| 1193 |
-
question: str,
|
| 1194 |
-
evidence: str,
|
| 1195 |
-
first_answer: str,
|
| 1196 |
-
second_answer: str,
|
| 1197 |
-
media=None,
|
| 1198 |
-
):
|
| 1199 |
prompt = f"""
|
| 1200 |
Two independent attempts produced different answers.
|
| 1201 |
|
|
@@ -1211,18 +1240,12 @@ CANDIDATE A:
|
|
| 1211 |
CANDIDATE B:
|
| 1212 |
{second_answer}
|
| 1213 |
|
| 1214 |
-
Re-
|
| 1215 |
-
|
| 1216 |
-
|
| 1217 |
-
different answer if both candidates are wrong.
|
| 1218 |
-
|
| 1219 |
-
Return only the exact requested answer.
|
| 1220 |
"""
|
| 1221 |
|
| 1222 |
-
contents = [
|
| 1223 |
-
prompt
|
| 1224 |
-
]
|
| 1225 |
-
|
| 1226 |
if media is not None:
|
| 1227 |
contents.append(media)
|
| 1228 |
|
|
@@ -1230,56 +1253,32 @@ Return only the exact requested answer.
|
|
| 1230 |
model=GEMINI_MODEL,
|
| 1231 |
contents=contents,
|
| 1232 |
config=types.GenerateContentConfig(
|
| 1233 |
-
system_instruction=
|
| 1234 |
-
self.system_instruction(),
|
| 1235 |
max_output_tokens=2048,
|
| 1236 |
-
thinking_config=
|
| 1237 |
-
|
| 1238 |
-
|
| 1239 |
-
),
|
| 1240 |
-
response_mime_type=
|
| 1241 |
-
"application/json",
|
| 1242 |
-
response_schema=
|
| 1243 |
-
AnswerResponse,
|
| 1244 |
),
|
| 1245 |
)
|
| 1246 |
|
| 1247 |
if response.parsed:
|
| 1248 |
-
if isinstance(
|
| 1249 |
-
response.parsed,
|
| 1250 |
-
AnswerResponse,
|
| 1251 |
-
):
|
| 1252 |
parsed = response.parsed
|
| 1253 |
else:
|
| 1254 |
-
parsed = AnswerResponse.model_validate(
|
| 1255 |
-
response.parsed
|
| 1256 |
-
)
|
| 1257 |
-
|
| 1258 |
else:
|
| 1259 |
-
parsed = AnswerResponse.model_validate_json(
|
| 1260 |
-
response.text
|
| 1261 |
-
)
|
| 1262 |
|
| 1263 |
-
return clean_answer(
|
| 1264 |
-
parsed.answer
|
| 1265 |
-
)
|
| 1266 |
|
| 1267 |
-
def solve_with_consensus(
|
| 1268 |
-
self,
|
| 1269 |
-
question: str,
|
| 1270 |
-
evidence: str,
|
| 1271 |
-
media=None,
|
| 1272 |
-
):
|
| 1273 |
first = self.generate_answer(
|
| 1274 |
question,
|
| 1275 |
evidence,
|
| 1276 |
media,
|
| 1277 |
independent=False,
|
| 1278 |
)
|
| 1279 |
-
|
| 1280 |
-
print(
|
| 1281 |
-
f"First model answer: {first}"
|
| 1282 |
-
)
|
| 1283 |
|
| 1284 |
time.sleep(1.0)
|
| 1285 |
|
|
@@ -1289,26 +1288,13 @@ Return only the exact requested answer.
|
|
| 1289 |
media,
|
| 1290 |
independent=True,
|
| 1291 |
)
|
|
|
|
| 1292 |
|
| 1293 |
-
|
| 1294 |
-
|
| 1295 |
-
)
|
| 1296 |
-
|
| 1297 |
-
if (
|
| 1298 |
-
normalized_answer(first)
|
| 1299 |
-
== normalized_answer(second)
|
| 1300 |
-
):
|
| 1301 |
-
print(
|
| 1302 |
-
"Both model attempts agree."
|
| 1303 |
-
)
|
| 1304 |
-
|
| 1305 |
return first
|
| 1306 |
|
| 1307 |
-
print(
|
| 1308 |
-
"Model attempts disagree. "
|
| 1309 |
-
"Running adjudication."
|
| 1310 |
-
)
|
| 1311 |
-
|
| 1312 |
time.sleep(1.0)
|
| 1313 |
|
| 1314 |
final = self.adjudicate(
|
|
@@ -1318,580 +1304,280 @@ Return only the exact requested answer.
|
|
| 1318 |
second,
|
| 1319 |
media,
|
| 1320 |
)
|
| 1321 |
-
|
| 1322 |
-
print(
|
| 1323 |
-
f"Adjudicated answer: {final}"
|
| 1324 |
-
)
|
| 1325 |
-
|
| 1326 |
return final
|
| 1327 |
|
| 1328 |
-
def solve_task(
|
| 1329 |
-
self,
|
| 1330 |
-
task: dict,
|
| 1331 |
-
):
|
| 1332 |
task_id = task["task_id"]
|
| 1333 |
question = task["question"]
|
| 1334 |
file_name = task["file_name"]
|
| 1335 |
|
| 1336 |
print("")
|
| 1337 |
-
print(
|
| 1338 |
-
|
| 1339 |
-
|
| 1340 |
-
|
| 1341 |
-
|
| 1342 |
-
self.
|
| 1343 |
-
|
| 1344 |
-
|
| 1345 |
-
|
| 1346 |
-
|
| 1347 |
-
|
| 1348 |
-
|
| 1349 |
-
|
| 1350 |
-
table_answer = (
|
| 1351 |
-
self.solve_operation_table(
|
| 1352 |
-
question
|
| 1353 |
-
)
|
| 1354 |
-
)
|
| 1355 |
-
|
| 1356 |
-
if table_answer is not None:
|
| 1357 |
-
return table_answer
|
| 1358 |
|
| 1359 |
path = None
|
| 1360 |
-
|
| 1361 |
if file_name:
|
| 1362 |
-
path = self.download_attachment(
|
| 1363 |
-
task_id,
|
| 1364 |
-
file_name,
|
| 1365 |
-
)
|
| 1366 |
-
|
| 1367 |
-
if (
|
| 1368 |
-
path is not None
|
| 1369 |
-
and path.suffix.lower() == ".py"
|
| 1370 |
-
):
|
| 1371 |
-
python_answer = (
|
| 1372 |
-
self.solve_python_file(
|
| 1373 |
-
path,
|
| 1374 |
-
question,
|
| 1375 |
-
)
|
| 1376 |
-
)
|
| 1377 |
|
| 1378 |
-
|
| 1379 |
-
|
|
|
|
|
|
|
| 1380 |
|
| 1381 |
-
if (
|
| 1382 |
-
|
| 1383 |
-
|
| 1384 |
-
|
| 1385 |
-
):
|
| 1386 |
-
excel_answer = (
|
| 1387 |
-
self.solve_excel_food_sales(
|
| 1388 |
-
path,
|
| 1389 |
-
question,
|
| 1390 |
-
)
|
| 1391 |
-
)
|
| 1392 |
|
| 1393 |
-
|
| 1394 |
-
|
| 1395 |
-
|
| 1396 |
-
|
| 1397 |
-
self.fetch_youtube_transcript(
|
| 1398 |
-
question
|
| 1399 |
-
)
|
| 1400 |
-
)
|
| 1401 |
|
|
|
|
| 1402 |
if youtube_transcript:
|
| 1403 |
evidence = (
|
| 1404 |
-
"The following is the transcript "
|
| 1405 |
-
"
|
| 1406 |
-
"in the question:\n\n"
|
| 1407 |
-
+ youtube_transcript
|
| 1408 |
-
)
|
| 1409 |
-
|
| 1410 |
-
return self.solve_with_consensus(
|
| 1411 |
-
question,
|
| 1412 |
-
evidence,
|
| 1413 |
-
)
|
| 1414 |
-
|
| 1415 |
-
if (
|
| 1416 |
-
path is not None
|
| 1417 |
-
and path.suffix.lower()
|
| 1418 |
-
in MEDIA_EXTENSIONS
|
| 1419 |
-
):
|
| 1420 |
-
print(
|
| 1421 |
-
f"Uploading media: {file_name}"
|
| 1422 |
-
)
|
| 1423 |
-
|
| 1424 |
-
media = self.client.files.upload(
|
| 1425 |
-
file=str(path)
|
| 1426 |
)
|
|
|
|
| 1427 |
|
| 1428 |
-
|
| 1429 |
-
|
| 1430 |
-
)
|
|
|
|
| 1431 |
|
| 1432 |
evidence = (
|
| 1433 |
-
"The actual media attachment referenced "
|
| 1434 |
-
"
|
| 1435 |
-
"Inspect the attachment itself carefully. "
|
| 1436 |
f"Filename: {file_name}"
|
| 1437 |
)
|
|
|
|
| 1438 |
|
| 1439 |
-
|
| 1440 |
-
|
| 1441 |
-
evidence,
|
| 1442 |
-
media,
|
| 1443 |
-
)
|
| 1444 |
-
|
| 1445 |
-
if (
|
| 1446 |
-
path is not None
|
| 1447 |
-
and path.suffix.lower()
|
| 1448 |
-
in {".xlsx", ".xlsm"}
|
| 1449 |
-
):
|
| 1450 |
-
workbook_text = (
|
| 1451 |
-
self.workbook_to_text(
|
| 1452 |
-
path
|
| 1453 |
-
)
|
| 1454 |
-
)
|
| 1455 |
-
|
| 1456 |
evidence = (
|
| 1457 |
-
"The following data was extracted from "
|
| 1458 |
-
"the attached workbook:\n\n"
|
| 1459 |
+ workbook_text
|
| 1460 |
)
|
|
|
|
| 1461 |
|
| 1462 |
-
|
| 1463 |
-
|
| 1464 |
-
evidence,
|
| 1465 |
-
)
|
| 1466 |
-
|
| 1467 |
-
if (
|
| 1468 |
-
path is not None
|
| 1469 |
-
and path.suffix.lower()
|
| 1470 |
-
in {
|
| 1471 |
-
".txt",
|
| 1472 |
-
".csv",
|
| 1473 |
-
".json",
|
| 1474 |
-
".md",
|
| 1475 |
-
".py",
|
| 1476 |
-
}
|
| 1477 |
-
):
|
| 1478 |
-
file_text = path.read_text(
|
| 1479 |
-
encoding="utf-8",
|
| 1480 |
-
errors="replace",
|
| 1481 |
-
)[:70000]
|
| 1482 |
-
|
| 1483 |
evidence = (
|
| 1484 |
-
"The following is the content of the "
|
| 1485 |
-
"attached file:\n\n"
|
| 1486 |
+ file_text
|
| 1487 |
)
|
|
|
|
| 1488 |
|
| 1489 |
-
|
| 1490 |
-
|
| 1491 |
-
|
| 1492 |
-
|
| 1493 |
-
|
| 1494 |
-
print(
|
| 1495 |
-
"Running targeted web research."
|
| 1496 |
-
)
|
| 1497 |
-
|
| 1498 |
-
research = self.web_research(
|
| 1499 |
-
question
|
| 1500 |
-
)
|
| 1501 |
-
|
| 1502 |
-
print(
|
| 1503 |
-
f"Web research collected: "
|
| 1504 |
-
f"{len(research)} chars"
|
| 1505 |
-
)
|
| 1506 |
-
|
| 1507 |
-
return self.solve_with_consensus(
|
| 1508 |
-
question,
|
| 1509 |
-
research,
|
| 1510 |
-
)
|
| 1511 |
|
| 1512 |
|
| 1513 |
-
def build_results_table(
|
| 1514 |
-
tasks: list[dict],
|
| 1515 |
-
cache: dict,
|
| 1516 |
-
):
|
| 1517 |
rows = []
|
| 1518 |
|
| 1519 |
for task in tasks:
|
| 1520 |
-
task_id = task[
|
| 1521 |
-
"task_id"
|
| 1522 |
-
]
|
| 1523 |
-
|
| 1524 |
rows.append(
|
| 1525 |
{
|
| 1526 |
-
"Task ID":
|
| 1527 |
-
|
| 1528 |
-
"
|
| 1529 |
-
|
| 1530 |
-
|
| 1531 |
-
],
|
| 1532 |
-
"File":
|
| 1533 |
-
task[
|
| 1534 |
-
"file_name"
|
| 1535 |
-
],
|
| 1536 |
-
"Answer":
|
| 1537 |
-
cache.get(
|
| 1538 |
-
task_id,
|
| 1539 |
-
"",
|
| 1540 |
-
),
|
| 1541 |
-
"Status":
|
| 1542 |
-
(
|
| 1543 |
-
"READY"
|
| 1544 |
-
if task_id in cache
|
| 1545 |
-
else "PENDING"
|
| 1546 |
-
),
|
| 1547 |
}
|
| 1548 |
)
|
| 1549 |
|
| 1550 |
-
return pd.DataFrame(
|
| 1551 |
-
rows
|
| 1552 |
-
)
|
| 1553 |
|
| 1554 |
|
| 1555 |
-
def run_and_submit_all(
|
| 1556 |
-
profile: gr.OAuthProfile | None,
|
| 1557 |
-
):
|
| 1558 |
if not profile:
|
| 1559 |
-
return
|
| 1560 |
-
"Please Login to Hugging Face.",
|
| 1561 |
-
None,
|
| 1562 |
-
)
|
| 1563 |
-
|
| 1564 |
-
username = str(
|
| 1565 |
-
profile.username
|
| 1566 |
-
).strip()
|
| 1567 |
|
| 1568 |
-
|
| 1569 |
-
|
| 1570 |
-
)
|
| 1571 |
|
| 1572 |
if not space_id:
|
| 1573 |
-
return
|
| 1574 |
-
"SPACE_ID environment variable is missing.",
|
| 1575 |
-
None,
|
| 1576 |
-
)
|
| 1577 |
-
|
| 1578 |
-
questions_url = (
|
| 1579 |
-
f"{DEFAULT_API_URL}/questions"
|
| 1580 |
-
)
|
| 1581 |
|
| 1582 |
-
|
| 1583 |
-
|
| 1584 |
-
)
|
| 1585 |
|
| 1586 |
try:
|
| 1587 |
response = requests.get(
|
| 1588 |
questions_url,
|
|
|
|
| 1589 |
timeout=30,
|
| 1590 |
)
|
| 1591 |
-
|
| 1592 |
response.raise_for_status()
|
| 1593 |
-
|
| 1594 |
-
questions_data = (
|
| 1595 |
-
response.json()
|
| 1596 |
-
)
|
| 1597 |
-
|
| 1598 |
except Exception as error:
|
| 1599 |
-
return
|
| 1600 |
-
f"Could not fetch questions: {error}",
|
| 1601 |
-
None,
|
| 1602 |
-
)
|
| 1603 |
|
| 1604 |
if not questions_data:
|
| 1605 |
-
return
|
| 1606 |
-
"No evaluation questions were returned.",
|
| 1607 |
-
None,
|
| 1608 |
-
)
|
| 1609 |
|
| 1610 |
tasks = []
|
| 1611 |
-
|
| 1612 |
for item in questions_data:
|
| 1613 |
-
task_id = item.get(
|
| 1614 |
-
|
| 1615 |
-
)
|
| 1616 |
-
|
| 1617 |
-
question = item.get(
|
| 1618 |
-
"question"
|
| 1619 |
-
)
|
| 1620 |
-
|
| 1621 |
-
file_name = (
|
| 1622 |
-
item.get(
|
| 1623 |
-
"file_name"
|
| 1624 |
-
)
|
| 1625 |
-
or ""
|
| 1626 |
-
)
|
| 1627 |
|
| 1628 |
-
if
|
| 1629 |
-
|
| 1630 |
-
or question is None
|
| 1631 |
-
):
|
| 1632 |
-
return (
|
| 1633 |
-
"The questions API returned "
|
| 1634 |
-
"an invalid task.",
|
| 1635 |
-
None,
|
| 1636 |
-
)
|
| 1637 |
|
| 1638 |
tasks.append(
|
| 1639 |
{
|
| 1640 |
-
"task_id":
|
| 1641 |
-
|
| 1642 |
-
"
|
| 1643 |
-
question,
|
| 1644 |
-
"file_name":
|
| 1645 |
-
file_name,
|
| 1646 |
}
|
| 1647 |
)
|
| 1648 |
|
| 1649 |
try:
|
| 1650 |
agent = BasicAgent()
|
| 1651 |
-
|
| 1652 |
except Exception as error:
|
| 1653 |
-
return
|
| 1654 |
-
f"Could not initialize agent: {error}",
|
| 1655 |
-
None,
|
| 1656 |
-
)
|
| 1657 |
|
| 1658 |
cache = load_cache()
|
| 1659 |
-
|
| 1660 |
-
valid_ids = {
|
| 1661 |
-
task["task_id"]
|
| 1662 |
-
for task in tasks
|
| 1663 |
-
}
|
| 1664 |
-
|
| 1665 |
cache = {
|
| 1666 |
task_id: answer
|
| 1667 |
-
for task_id, answer
|
| 1668 |
-
in cache.items()
|
| 1669 |
if task_id in valid_ids
|
| 1670 |
}
|
| 1671 |
-
|
| 1672 |
save_cache(cache)
|
| 1673 |
|
| 1674 |
-
print(
|
| 1675 |
-
|
| 1676 |
-
)
|
| 1677 |
-
|
| 1678 |
-
print(
|
| 1679 |
-
f"Already solved: {len(cache)}"
|
| 1680 |
-
)
|
| 1681 |
-
|
| 1682 |
-
print(
|
| 1683 |
-
f"Remaining: {len(tasks) - len(cache)}"
|
| 1684 |
-
)
|
| 1685 |
|
| 1686 |
try:
|
| 1687 |
-
for index, task in enumerate(
|
| 1688 |
-
|
| 1689 |
-
start=1,
|
| 1690 |
-
):
|
| 1691 |
-
task_id = task[
|
| 1692 |
-
"task_id"
|
| 1693 |
-
]
|
| 1694 |
|
| 1695 |
if task_id in cache:
|
| 1696 |
print(
|
| 1697 |
-
f"Skipping cached task "
|
| 1698 |
-
f"{index}/{len(tasks)}: "
|
| 1699 |
-
f"{task_id}"
|
| 1700 |
)
|
| 1701 |
continue
|
| 1702 |
|
| 1703 |
print("")
|
| 1704 |
-
print(
|
| 1705 |
-
|
| 1706 |
-
f"{index}/{len(tasks)}"
|
| 1707 |
-
)
|
| 1708 |
-
|
| 1709 |
-
answer = agent.solve_task(
|
| 1710 |
-
task
|
| 1711 |
-
)
|
| 1712 |
-
|
| 1713 |
-
answer = clean_answer(
|
| 1714 |
-
answer
|
| 1715 |
-
)
|
| 1716 |
|
| 1717 |
if not answer:
|
| 1718 |
-
raise RuntimeError(
|
| 1719 |
-
f"Task {task_id} "
|
| 1720 |
-
"produced an empty answer."
|
| 1721 |
-
)
|
| 1722 |
-
|
| 1723 |
-
cache[
|
| 1724 |
-
task_id
|
| 1725 |
-
] = answer
|
| 1726 |
-
|
| 1727 |
-
save_cache(
|
| 1728 |
-
cache
|
| 1729 |
-
)
|
| 1730 |
-
|
| 1731 |
-
print(
|
| 1732 |
-
f"Saved answer for "
|
| 1733 |
-
f"{task_id}: {answer}"
|
| 1734 |
-
)
|
| 1735 |
|
|
|
|
|
|
|
|
|
|
| 1736 |
time.sleep(1.0)
|
| 1737 |
|
| 1738 |
except Exception as error:
|
| 1739 |
print(
|
| 1740 |
"Evaluation paused: "
|
| 1741 |
-
f"{type(error).__name__}: "
|
| 1742 |
-
f"{error}"
|
| 1743 |
)
|
| 1744 |
-
|
| 1745 |
return (
|
| 1746 |
-
|
| 1747 |
-
|
| 1748 |
-
|
| 1749 |
-
|
| 1750 |
-
|
| 1751 |
-
|
| 1752 |
-
f"{len(tasks) - len(cache)}\n\n"
|
| 1753 |
-
f"Error: "
|
| 1754 |
-
f"{type(error).__name__}: "
|
| 1755 |
-
f"{error}\n\n"
|
| 1756 |
-
"Click the same button again "
|
| 1757 |
-
"to resume from the saved answers."
|
| 1758 |
-
),
|
| 1759 |
-
build_results_table(
|
| 1760 |
-
tasks,
|
| 1761 |
-
cache,
|
| 1762 |
-
),
|
| 1763 |
)
|
| 1764 |
|
| 1765 |
missing = [
|
| 1766 |
task["task_id"]
|
| 1767 |
for task in tasks
|
| 1768 |
-
if task["task_id"]
|
| 1769 |
-
not in cache
|
| 1770 |
]
|
| 1771 |
|
| 1772 |
if missing:
|
| 1773 |
return (
|
| 1774 |
-
|
| 1775 |
-
|
| 1776 |
-
|
| 1777 |
-
f"Saved answers: "
|
| 1778 |
-
f"{len(cache)}/{len(tasks)}"
|
| 1779 |
-
),
|
| 1780 |
-
build_results_table(
|
| 1781 |
-
tasks,
|
| 1782 |
-
cache,
|
| 1783 |
-
),
|
| 1784 |
)
|
| 1785 |
|
| 1786 |
answers_payload = [
|
| 1787 |
{
|
| 1788 |
-
"task_id":
|
| 1789 |
-
|
| 1790 |
-
"submitted_answer":
|
| 1791 |
-
cache[
|
| 1792 |
-
task["task_id"]
|
| 1793 |
-
],
|
| 1794 |
}
|
| 1795 |
for task in tasks
|
| 1796 |
]
|
| 1797 |
|
| 1798 |
-
agent_code =
|
| 1799 |
-
"https://huggingface.co/"
|
| 1800 |
-
f"spaces/{space_id}/tree/main"
|
| 1801 |
-
)
|
| 1802 |
-
|
| 1803 |
submission_data = {
|
| 1804 |
-
"username":
|
| 1805 |
-
|
| 1806 |
-
"
|
| 1807 |
-
agent_code,
|
| 1808 |
-
"answers":
|
| 1809 |
-
answers_payload,
|
| 1810 |
}
|
| 1811 |
|
| 1812 |
print("")
|
| 1813 |
-
print(
|
| 1814 |
-
|
| 1815 |
-
)
|
| 1816 |
-
|
| 1817 |
-
print(
|
| 1818 |
-
"Submitting answers..."
|
| 1819 |
-
)
|
| 1820 |
|
| 1821 |
try:
|
| 1822 |
response = requests.post(
|
| 1823 |
submit_url,
|
| 1824 |
json=submission_data,
|
|
|
|
| 1825 |
timeout=120,
|
| 1826 |
)
|
| 1827 |
-
|
| 1828 |
response.raise_for_status()
|
| 1829 |
-
|
| 1830 |
-
result = (
|
| 1831 |
-
response.json()
|
| 1832 |
-
)
|
| 1833 |
-
|
| 1834 |
except Exception as error:
|
| 1835 |
-
print(
|
| 1836 |
-
f"Submission failed: {error}"
|
| 1837 |
-
)
|
| 1838 |
-
|
| 1839 |
return (
|
| 1840 |
-
|
| 1841 |
-
|
| 1842 |
-
|
| 1843 |
-
|
| 1844 |
-
"Click the same button again. "
|
| 1845 |
-
"The saved answers will not "
|
| 1846 |
-
"be recomputed."
|
| 1847 |
-
),
|
| 1848 |
-
build_results_table(
|
| 1849 |
-
tasks,
|
| 1850 |
-
cache,
|
| 1851 |
-
),
|
| 1852 |
)
|
| 1853 |
|
| 1854 |
final_status = (
|
| 1855 |
"Submission Successful!\n"
|
| 1856 |
-
f"User: "
|
| 1857 |
-
f"{result.get('
|
| 1858 |
-
f"
|
| 1859 |
-
f"{result.get('
|
| 1860 |
-
f"("
|
| 1861 |
-
f"{result.get('correct_count', '?')}/"
|
| 1862 |
-
f"{result.get('total_attempted', '?')} "
|
| 1863 |
-
f"correct)\n"
|
| 1864 |
-
f"Message: "
|
| 1865 |
-
f"{result.get('message', '')}"
|
| 1866 |
-
)
|
| 1867 |
-
|
| 1868 |
-
print(
|
| 1869 |
-
final_status
|
| 1870 |
)
|
| 1871 |
|
| 1872 |
-
|
| 1873 |
-
|
| 1874 |
-
build_results_table(
|
| 1875 |
-
tasks,
|
| 1876 |
-
cache,
|
| 1877 |
-
),
|
| 1878 |
-
)
|
| 1879 |
|
| 1880 |
|
| 1881 |
with gr.Blocks() as demo:
|
| 1882 |
-
gr.Markdown(
|
| 1883 |
-
"# Basic Agent Evaluation Runner"
|
| 1884 |
-
)
|
| 1885 |
-
|
| 1886 |
gr.Markdown(
|
| 1887 |
"""
|
| 1888 |
**Instructions:**
|
| 1889 |
|
| 1890 |
1. Log in to your Hugging Face account.
|
| 1891 |
2. Click **Run / Resume Evaluation & Submit All Answers**.
|
| 1892 |
-
3. Deterministic tasks are solved locally where possible.
|
| 1893 |
4. Research and media tasks are independently checked before acceptance.
|
| 1894 |
-
5. Completed answers are cached
|
| 1895 |
6. Submission occurs only after all 20 answers are ready.
|
| 1896 |
"""
|
| 1897 |
)
|
|
@@ -1903,73 +1589,43 @@ with gr.Blocks() as demo:
|
|
| 1903 |
)
|
| 1904 |
|
| 1905 |
status_output = gr.Textbox(
|
| 1906 |
-
label=
|
| 1907 |
-
"Run Status / Submission Result"
|
| 1908 |
-
),
|
| 1909 |
lines=10,
|
| 1910 |
interactive=False,
|
| 1911 |
)
|
| 1912 |
|
| 1913 |
results_table = gr.DataFrame(
|
| 1914 |
-
label=
|
| 1915 |
-
"Questions and Agent Answers"
|
| 1916 |
-
),
|
| 1917 |
wrap=True,
|
| 1918 |
)
|
| 1919 |
|
| 1920 |
run_button.click(
|
| 1921 |
fn=run_and_submit_all,
|
| 1922 |
-
outputs=[
|
| 1923 |
-
status_output,
|
| 1924 |
-
results_table,
|
| 1925 |
-
],
|
| 1926 |
)
|
| 1927 |
|
| 1928 |
|
| 1929 |
if __name__ == "__main__":
|
| 1930 |
-
print(
|
| 1931 |
-
"\n"
|
| 1932 |
-
+ "-" * 30
|
| 1933 |
-
+ " App Starting "
|
| 1934 |
-
+ "-" * 30
|
| 1935 |
-
)
|
| 1936 |
|
| 1937 |
-
space_host = os.getenv(
|
| 1938 |
-
|
| 1939 |
-
)
|
| 1940 |
-
|
| 1941 |
-
space_id = os.getenv(
|
| 1942 |
-
"SPACE_ID"
|
| 1943 |
-
)
|
| 1944 |
|
| 1945 |
if space_host:
|
| 1946 |
-
print(
|
| 1947 |
-
f"SPACE_HOST found: "
|
| 1948 |
-
f"{space_host}"
|
| 1949 |
-
)
|
| 1950 |
|
| 1951 |
if space_id:
|
| 1952 |
-
print(
|
| 1953 |
-
f"SPACE_ID found: "
|
| 1954 |
-
f"{space_id}"
|
| 1955 |
-
)
|
| 1956 |
-
|
| 1957 |
print(
|
| 1958 |
"Repo Tree URL: "
|
| 1959 |
-
"https://huggingface.co/"
|
| 1960 |
-
f"spaces/{space_id}/tree/main"
|
| 1961 |
)
|
| 1962 |
|
| 1963 |
-
print(
|
| 1964 |
-
|
| 1965 |
-
)
|
| 1966 |
-
|
| 1967 |
-
print(
|
| 1968 |
-
"Launching Gradio Interface..."
|
| 1969 |
-
)
|
| 1970 |
|
| 1971 |
demo.launch(
|
| 1972 |
debug=True,
|
| 1973 |
share=False,
|
| 1974 |
ssr_mode=False,
|
| 1975 |
-
)
|
|
|
|
| 4 |
import subprocess
|
| 5 |
import sys
|
| 6 |
import time
|
| 7 |
+
from io import StringIO
|
| 8 |
from pathlib import Path
|
| 9 |
from urllib.parse import parse_qs, urlparse
|
| 10 |
|
| 11 |
+
import chess
|
| 12 |
import gradio as gr
|
| 13 |
import pandas as pd
|
| 14 |
import requests
|
| 15 |
import spaces
|
| 16 |
+
from bs4 import BeautifulSoup
|
| 17 |
from ddgs import DDGS
|
| 18 |
from google import genai
|
| 19 |
from google.genai import types
|
|
|
|
| 29 |
|
| 30 |
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
|
| 31 |
GEMINI_MODEL = "gemini-3.5-flash-lite"
|
| 32 |
+
USER_AGENT = "Mozilla/5.0 GAIAStudentAgent/1.0"
|
| 33 |
|
| 34 |
GAIA_ATTACHMENT_FALLBACK_BASE = (
|
| 35 |
"https://huggingface.co/spaces/bstraehle/gaia/"
|
| 36 |
"resolve/809adb7a8f1747286eac400b8675dffdd973fbb9/data"
|
| 37 |
)
|
| 38 |
|
| 39 |
+
ATTACHMENT_DIR = Path("/tmp/gaia_attachments_v7")
|
| 40 |
ATTACHMENT_DIR.mkdir(parents=True, exist_ok=True)
|
| 41 |
|
| 42 |
+
if Path("/data").exists() and os.access("/data", os.W_OK):
|
| 43 |
+
CACHE_FILE = Path("/data/gaia_answer_cache_v7.json")
|
| 44 |
+
else:
|
| 45 |
+
CACHE_FILE = Path("/tmp/gaia_answer_cache_v7.json")
|
| 46 |
|
| 47 |
MEDIA_EXTENSIONS = {
|
| 48 |
".png",
|
|
|
|
| 56 |
".ogg",
|
| 57 |
}
|
| 58 |
|
| 59 |
+
LIBRETEXT_EXERCISES_URL = (
|
| 60 |
+
"https://chem.libretexts.org/Bookshelves/Introductory_Chemistry/"
|
| 61 |
+
"Introductory_Chemistry_(LibreTexts)/01%3A_The_Chemical_World/"
|
| 62 |
+
"1.E%3A_Exercises"
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
OLYMPEDIA_1928_COUNTRIES_URL = "https://www.olympedia.org/editions/9/countries"
|
| 66 |
+
FIGHTERS_2023_ROSTER_URL = "https://npbdata.jp/2023/teams/fighters"
|
| 67 |
+
LICHESS_CLOUD_EVAL_URL = "https://lichess.org/api/cloud-eval"
|
| 68 |
+
|
| 69 |
|
| 70 |
class AnswerResponse(BaseModel):
|
| 71 |
answer: str
|
|
|
|
| 75 |
queries: list[str]
|
| 76 |
|
| 77 |
|
| 78 |
+
class ChessPosition(BaseModel):
|
| 79 |
+
fen: str
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
class RomanSurnamePair(BaseModel):
|
| 83 |
+
before: str
|
| 84 |
+
after: str
|
| 85 |
+
|
| 86 |
+
|
| 87 |
def load_cache() -> dict:
|
| 88 |
if not CACHE_FILE.exists():
|
| 89 |
return {}
|
| 90 |
|
| 91 |
try:
|
| 92 |
+
data = json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
|
|
|
|
|
|
|
|
|
| 93 |
if isinstance(data, dict):
|
| 94 |
return data
|
|
|
|
| 95 |
except Exception as error:
|
| 96 |
print(f"Could not load cache: {error}")
|
| 97 |
|
|
|
|
| 100 |
|
| 101 |
def save_cache(cache: dict):
|
| 102 |
CACHE_FILE.write_text(
|
| 103 |
+
json.dumps(cache, ensure_ascii=False, indent=2),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
encoding="utf-8",
|
| 105 |
)
|
| 106 |
|
|
|
|
| 128 |
|
| 129 |
if answer.startswith("```") and answer.endswith("```"):
|
| 130 |
answer = answer[3:-3].strip()
|
|
|
|
| 131 |
first_line, separator, remainder = answer.partition("\n")
|
| 132 |
+
if separator and first_line.strip().lower() in {
|
| 133 |
+
"text",
|
| 134 |
+
"plaintext",
|
| 135 |
+
"python",
|
| 136 |
+
"json",
|
| 137 |
+
}:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 138 |
answer = remainder.strip()
|
| 139 |
|
| 140 |
return answer.strip("`").strip()
|
| 141 |
|
| 142 |
|
| 143 |
def normalized_answer(answer: str) -> str:
|
| 144 |
+
return re.sub(r"\s+", " ", clean_answer(answer)).strip().casefold()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
|
| 146 |
|
| 147 |
def extract_youtube_id(text: str):
|
| 148 |
+
urls = re.findall(r"https?://[^\s)>]+", text)
|
|
|
|
|
|
|
|
|
|
| 149 |
|
| 150 |
for url in urls:
|
| 151 |
url = url.rstrip(".,;!?")
|
|
|
|
| 152 |
try:
|
| 153 |
parsed = urlparse(url)
|
|
|
|
| 154 |
if "youtube.com" in parsed.netloc:
|
| 155 |
+
video_id = parse_qs(parsed.query).get("v")
|
|
|
|
|
|
|
|
|
|
| 156 |
if video_id:
|
| 157 |
return video_id[0]
|
|
|
|
| 158 |
if "youtu.be" in parsed.netloc:
|
| 159 |
value = parsed.path.strip("/")
|
|
|
|
| 160 |
if value:
|
| 161 |
return value
|
|
|
|
| 162 |
except Exception:
|
| 163 |
continue
|
| 164 |
|
| 165 |
return None
|
| 166 |
|
| 167 |
|
| 168 |
+
def html_to_text(html: str) -> str:
|
| 169 |
+
soup = BeautifulSoup(html, "html.parser")
|
| 170 |
+
for element in soup(["script", "style", "noscript"]):
|
| 171 |
+
element.decompose()
|
| 172 |
+
return " ".join(soup.stripped_strings)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
class BasicAgent:
|
| 176 |
def __init__(self):
|
| 177 |
api_key = os.getenv("GEMINI_API_KEY")
|
|
|
|
| 178 |
if not api_key:
|
| 179 |
+
raise RuntimeError("GEMINI_API_KEY secret is not configured.")
|
|
|
|
|
|
|
| 180 |
|
| 181 |
self.client = genai.Client(
|
| 182 |
api_key=api_key,
|
|
|
|
| 187 |
exp_base=2.0,
|
| 188 |
max_delay=30.0,
|
| 189 |
jitter=1.0,
|
| 190 |
+
http_status_codes=[408, 500, 502, 503, 504],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
)
|
| 192 |
),
|
| 193 |
)
|
| 194 |
|
| 195 |
+
print(f"BasicAgent initialized with {GEMINI_MODEL}.")
|
| 196 |
+
|
| 197 |
+
def get(self, url: str, timeout: int = 30, params=None):
|
| 198 |
+
return requests.get(
|
| 199 |
+
url,
|
| 200 |
+
params=params,
|
| 201 |
+
headers={"User-Agent": USER_AGENT},
|
| 202 |
+
timeout=timeout,
|
| 203 |
)
|
| 204 |
|
| 205 |
+
def download_attachment(self, task_id: str, file_name: str) -> Path:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
safe_name = os.path.basename(file_name)
|
|
|
|
| 207 |
if not safe_name:
|
| 208 |
+
raise RuntimeError("Attachment file name is empty.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
|
| 210 |
+
destination = ATTACHMENT_DIR / f"{task_id}_{safe_name}"
|
| 211 |
+
if destination.exists() and destination.stat().st_size > 0:
|
| 212 |
+
print(f"Using cached attachment: {safe_name}")
|
| 213 |
return destination
|
| 214 |
|
| 215 |
+
official_url = f"{DEFAULT_API_URL}/files/{task_id}"
|
|
|
|
|
|
|
|
|
|
| 216 |
official_error = None
|
| 217 |
|
| 218 |
try:
|
| 219 |
+
print(f"Trying official attachment endpoint: {official_url}")
|
| 220 |
+
response = self.get(official_url, timeout=30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
response.raise_for_status()
|
|
|
|
| 222 |
if not response.content:
|
| 223 |
+
raise RuntimeError("Official endpoint returned an empty file.")
|
| 224 |
+
destination.write_bytes(response.content)
|
| 225 |
+
print(f"Downloaded attachment from official endpoint: {safe_name}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
return destination
|
|
|
|
| 227 |
except Exception as error:
|
| 228 |
official_error = error
|
|
|
|
| 229 |
print(
|
| 230 |
"Official attachment download failed: "
|
| 231 |
f"{type(error).__name__}: {error}"
|
| 232 |
)
|
| 233 |
|
| 234 |
fallback_url = (
|
| 235 |
+
f"{GAIA_ATTACHMENT_FALLBACK_BASE}/{safe_name}?download=true"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
)
|
| 237 |
+
print(f"Trying attachment fallback: {fallback_url}")
|
| 238 |
|
| 239 |
try:
|
| 240 |
+
response = self.get(fallback_url, timeout=60)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
response.raise_for_status()
|
|
|
|
| 242 |
if not response.content:
|
| 243 |
+
raise RuntimeError("Fallback returned an empty file.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
|
| 245 |
+
content_type = response.headers.get("content-type", "").lower()
|
| 246 |
+
suffix = Path(safe_name).suffix.lower()
|
| 247 |
+
if "text/html" in content_type and suffix in MEDIA_EXTENSIONS:
|
|
|
|
| 248 |
raise RuntimeError(
|
| 249 |
+
"Fallback returned HTML instead of the requested attachment."
|
|
|
|
| 250 |
)
|
| 251 |
|
| 252 |
+
destination.write_bytes(response.content)
|
| 253 |
+
print(f"Downloaded attachment from fallback: {safe_name}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
return destination
|
|
|
|
| 255 |
except Exception as fallback_error:
|
| 256 |
raise RuntimeError(
|
| 257 |
"Unable to download attachment. "
|
|
|
|
| 259 |
f"Fallback error: {fallback_error}"
|
| 260 |
)
|
| 261 |
|
| 262 |
+
def solve_reversed_text(self, question: str):
|
|
|
|
|
|
|
|
|
|
| 263 |
reversed_question = question[::-1]
|
|
|
|
| 264 |
lowered = reversed_question.lower()
|
| 265 |
|
| 266 |
+
if "opposite of the word" not in lowered and "opposite of" not in lowered:
|
|
|
|
|
|
|
|
|
|
| 267 |
return None
|
| 268 |
|
| 269 |
+
target = None
|
| 270 |
+
for pattern in [
|
| 271 |
r'opposite of the word\s+["\']([^"\']+)["\']',
|
| 272 |
r'opposite of\s+["\']([^"\']+)["\']',
|
| 273 |
+
]:
|
| 274 |
+
match = re.search(pattern, reversed_question, flags=re.IGNORECASE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 275 |
if match:
|
| 276 |
+
target = match.group(1).strip().lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 277 |
break
|
| 278 |
|
| 279 |
if not target:
|
|
|
|
| 295 |
}
|
| 296 |
|
| 297 |
answer = opposites.get(target)
|
|
|
|
| 298 |
if answer:
|
| 299 |
+
print(f"Local reversed-text solver: {answer}")
|
|
|
|
|
|
|
|
|
|
| 300 |
return answer
|
| 301 |
|
| 302 |
+
def solve_operation_table(self, question: str):
|
|
|
|
|
|
|
|
|
|
| 303 |
lowered = question.lower()
|
| 304 |
+
if "commutative" not in lowered or "|" not in question:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
return None
|
| 306 |
|
| 307 |
set_match = re.search(
|
|
|
|
| 309 |
question,
|
| 310 |
flags=re.IGNORECASE,
|
| 311 |
)
|
|
|
|
| 312 |
if not set_match:
|
| 313 |
return None
|
| 314 |
|
|
|
|
| 317 |
for value in set_match.group(1).split(",")
|
| 318 |
if value.strip()
|
| 319 |
]
|
|
|
|
| 320 |
if len(elements) < 2:
|
| 321 |
return None
|
| 322 |
|
| 323 |
+
tokens = [token.strip() for token in re.findall(r"\|([^|]*)", question)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 324 |
rows = {}
|
| 325 |
n = len(elements)
|
|
|
|
| 326 |
index = 0
|
| 327 |
|
| 328 |
while index < len(tokens):
|
| 329 |
+
if tokens[index] in elements and index + n < len(tokens):
|
| 330 |
+
candidate = tokens[index + 1:index + 1 + n]
|
| 331 |
+
if len(candidate) == n and all(value in elements for value in candidate):
|
| 332 |
+
rows[tokens[index]] = dict(zip(elements, candidate))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
index += n + 1
|
| 334 |
continue
|
|
|
|
| 335 |
index += 1
|
| 336 |
|
| 337 |
if len(rows) != n:
|
| 338 |
return None
|
| 339 |
|
| 340 |
involved = set()
|
|
|
|
| 341 |
for left in elements:
|
| 342 |
for right in elements:
|
| 343 |
+
if rows[left].get(right) != rows[right].get(left):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 344 |
involved.add(left)
|
| 345 |
involved.add(right)
|
| 346 |
|
| 347 |
if not involved:
|
| 348 |
return None
|
| 349 |
|
| 350 |
+
answer = ", ".join(sorted(involved))
|
| 351 |
+
print(f"Local operation-table solver: {answer}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
return answer
|
| 353 |
|
| 354 |
+
def solve_python_file(self, path: Path, question: str):
|
| 355 |
+
if path.suffix.lower() != ".py" or "numeric output" not in question.lower():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 356 |
return None
|
| 357 |
|
| 358 |
try:
|
| 359 |
completed = subprocess.run(
|
| 360 |
+
[sys.executable, "-I", str(path)],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 361 |
cwd=str(path.parent),
|
| 362 |
capture_output=True,
|
| 363 |
text=True,
|
| 364 |
timeout=15,
|
| 365 |
)
|
|
|
|
| 366 |
except Exception as error:
|
| 367 |
+
print(f"Local Python execution failed: {error}")
|
|
|
|
|
|
|
| 368 |
return None
|
| 369 |
|
| 370 |
stdout = completed.stdout.strip()
|
|
|
|
| 371 |
if not stdout:
|
| 372 |
+
print("Python script produced no stdout.")
|
|
|
|
|
|
|
| 373 |
return None
|
| 374 |
|
| 375 |
+
lines = [line.strip() for line in stdout.splitlines() if line.strip()]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
if not lines:
|
| 377 |
return None
|
| 378 |
|
|
|
|
|
|
|
| 379 |
numeric_matches = re.findall(
|
| 380 |
r"-?(?:\d+(?:\.\d+)?|\.\d+)",
|
| 381 |
+
lines[-1],
|
| 382 |
)
|
|
|
|
| 383 |
if not numeric_matches:
|
| 384 |
return None
|
| 385 |
|
| 386 |
answer = numeric_matches[-1]
|
| 387 |
+
print(f"Local Python solver: {answer}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
return answer
|
| 389 |
|
| 390 |
+
def solve_excel_food_sales(self, path: Path, question: str):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 391 |
lowered = question.lower()
|
|
|
|
| 392 |
if (
|
| 393 |
+
path.suffix.lower() not in {".xlsx", ".xlsm"}
|
|
|
|
| 394 |
or "sales" not in lowered
|
| 395 |
or "food" not in lowered
|
| 396 |
+
or ("drink" not in lowered and "beverage" not in lowered)
|
|
|
|
|
|
|
|
|
|
| 397 |
):
|
| 398 |
return None
|
| 399 |
|
|
|
|
| 435 |
|
| 436 |
try:
|
| 437 |
workbook = pd.ExcelFile(path)
|
|
|
|
| 438 |
grand_total = 0.0
|
| 439 |
found_food_columns = 0
|
| 440 |
found_drink_columns = 0
|
| 441 |
|
| 442 |
for sheet in workbook.sheet_names:
|
| 443 |
+
frame = pd.read_excel(path, sheet_name=sheet)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 444 |
if frame.empty:
|
| 445 |
continue
|
| 446 |
|
|
|
|
|
|
|
| 447 |
for column in frame.columns:
|
| 448 |
+
values = pd.to_numeric(frame[column], errors="coerce")
|
| 449 |
+
if not values.notna().any():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 450 |
continue
|
| 451 |
|
| 452 |
+
column_name = str(column).strip().lower()
|
| 453 |
+
if any(word in column_name for word in metadata_words):
|
| 454 |
+
continue
|
| 455 |
+
if any(word in column_name for word in drink_words):
|
| 456 |
found_drink_columns += 1
|
| 457 |
continue
|
| 458 |
|
| 459 |
+
grand_total += float(values.fillna(0).sum())
|
|
|
|
|
|
|
|
|
|
| 460 |
found_food_columns += 1
|
| 461 |
|
| 462 |
+
if found_food_columns == 0 or found_drink_columns == 0:
|
|
|
|
|
|
|
|
|
|
| 463 |
return None
|
| 464 |
|
| 465 |
answer = f"{grand_total:.2f}"
|
|
|
|
| 466 |
print(
|
| 467 |
+
"Local Excel solver: "
|
| 468 |
+
f"food columns={found_food_columns}, "
|
| 469 |
+
f"drink columns={found_drink_columns}, "
|
| 470 |
+
f"answer={answer}"
|
| 471 |
)
|
|
|
|
| 472 |
return answer
|
|
|
|
| 473 |
except Exception as error:
|
| 474 |
+
print(f"Local Excel calculation failed: {error}")
|
| 475 |
+
return None
|
|
|
|
| 476 |
|
| 477 |
+
def solve_libretexts_equine_vet(self, question: str):
|
| 478 |
+
lowered = question.lower()
|
| 479 |
+
if "equine veterinarian" not in lowered or "libretext" not in lowered:
|
| 480 |
return None
|
| 481 |
|
| 482 |
+
texts = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 483 |
|
| 484 |
+
try:
|
| 485 |
+
response = self.get(LIBRETEXT_EXERCISES_URL, timeout=45)
|
| 486 |
+
response.raise_for_status()
|
| 487 |
+
texts.append(html_to_text(response.text))
|
| 488 |
+
print("Fetched LibreTexts 1.E Exercises directly.")
|
| 489 |
+
except Exception as error:
|
| 490 |
+
print(f"Direct LibreTexts fetch failed: {error}")
|
| 491 |
+
|
| 492 |
+
search_queries = [
|
| 493 |
+
'site:chem.libretexts.org "horse doctor" "anthrax"',
|
| 494 |
+
'site:chem.libretexts.org "1.E: Exercises" "horse doctor"',
|
| 495 |
+
'"Around 1876" "horse doctor" anthrax',
|
| 496 |
+
]
|
| 497 |
+
|
| 498 |
+
for query in search_queries:
|
| 499 |
+
try:
|
| 500 |
+
results = DDGS(timeout=15).text(
|
| 501 |
+
query,
|
| 502 |
+
region="us-en",
|
| 503 |
+
safesearch="moderate",
|
| 504 |
+
max_results=6,
|
| 505 |
+
)
|
| 506 |
+
for result in results or []:
|
| 507 |
+
texts.append(
|
| 508 |
+
" ".join(
|
| 509 |
+
[
|
| 510 |
+
str(result.get("title") or ""),
|
| 511 |
+
str(result.get("body") or result.get("description") or ""),
|
| 512 |
+
]
|
| 513 |
+
)
|
| 514 |
+
)
|
| 515 |
+
except Exception as error:
|
| 516 |
+
print(f"LibreTexts search failed for {query}: {error}")
|
| 517 |
+
|
| 518 |
+
combined = "\n".join(texts)
|
| 519 |
+
patterns = [
|
| 520 |
+
r"horse doctor.{0,180}?named\s+([A-Z][A-Za-z'’-]+)",
|
| 521 |
+
r"named\s+([A-Z][A-Za-z'’-]+).{0,180}?cure for anthrax",
|
| 522 |
+
r"([A-Z][A-Za-z'’-]+).{0,120}?horse doctor",
|
| 523 |
+
]
|
| 524 |
+
|
| 525 |
+
for pattern in patterns:
|
| 526 |
+
match = re.search(pattern, combined, flags=re.IGNORECASE | re.DOTALL)
|
| 527 |
+
if match:
|
| 528 |
+
answer = match.group(1).strip().strip(".,;:()[]")
|
| 529 |
+
if answer and answer.casefold() not in {"a", "the", "an"}:
|
| 530 |
+
answer = answer[0].upper() + answer[1:]
|
| 531 |
+
print(f"Direct-source veterinarian solver: {answer}")
|
| 532 |
+
return answer
|
| 533 |
+
|
| 534 |
+
print("Direct-source veterinarian solver could not extract a surname.")
|
| 535 |
+
return None
|
| 536 |
+
|
| 537 |
+
def solve_1928_olympics(self, question: str):
|
| 538 |
+
lowered = question.lower()
|
| 539 |
+
if (
|
| 540 |
+
"1928 summer olympics" not in lowered
|
| 541 |
+
or "least number of athletes" not in lowered
|
| 542 |
+
or "ioc" not in lowered
|
| 543 |
+
):
|
| 544 |
+
return None
|
| 545 |
|
| 546 |
try:
|
| 547 |
+
response = self.get(OLYMPEDIA_1928_COUNTRIES_URL, timeout=45)
|
| 548 |
+
response.raise_for_status()
|
| 549 |
+
tables = pd.read_html(StringIO(response.text))
|
| 550 |
+
except Exception as error:
|
| 551 |
+
print(f"Olympedia table fetch failed: {error}")
|
| 552 |
+
return None
|
| 553 |
+
|
| 554 |
+
for frame in tables:
|
| 555 |
+
columns = [str(column).strip() for column in frame.columns]
|
| 556 |
+
frame.columns = columns
|
| 557 |
+
|
| 558 |
+
if "Country" not in columns or "Total" not in columns:
|
| 559 |
+
continue
|
| 560 |
+
|
| 561 |
+
code_column = None
|
| 562 |
+
for column in columns:
|
| 563 |
+
if column not in {"Country", "Men", "Women", "Total"}:
|
| 564 |
+
code_column = column
|
| 565 |
+
break
|
| 566 |
+
|
| 567 |
+
if code_column is None:
|
| 568 |
+
continue
|
| 569 |
+
|
| 570 |
+
working = frame.copy()
|
| 571 |
+
working["Total"] = pd.to_numeric(working["Total"], errors="coerce")
|
| 572 |
+
working = working.dropna(subset=["Country", "Total"])
|
| 573 |
+
working = working[
|
| 574 |
+
~working["Country"].astype(str).str.fullmatch("Total", case=False, na=False)
|
| 575 |
+
]
|
| 576 |
|
| 577 |
+
if working.empty:
|
| 578 |
+
continue
|
| 579 |
+
|
| 580 |
+
minimum = working["Total"].min()
|
| 581 |
+
candidates = working[working["Total"] == minimum].copy()
|
| 582 |
+
candidates["_sort"] = candidates["Country"].astype(str).str.casefold()
|
| 583 |
+
candidates = candidates.sort_values("_sort")
|
| 584 |
+
row = candidates.iloc[0]
|
| 585 |
+
answer = str(row[code_column]).strip()
|
| 586 |
+
|
| 587 |
+
if re.fullmatch(r"[A-Z]{3}", answer):
|
| 588 |
+
print(
|
| 589 |
+
"Deterministic Olympics solver: "
|
| 590 |
+
f"minimum={minimum}, country={row['Country']}, IOC={answer}"
|
| 591 |
)
|
| 592 |
+
return answer
|
| 593 |
+
|
| 594 |
+
print("Olympedia table was found but could not be parsed deterministically.")
|
| 595 |
+
return None
|
| 596 |
|
| 597 |
+
def extract_roman_surname_from_search(self, japanese_name: str):
|
| 598 |
+
queries = [
|
| 599 |
+
f'"{japanese_name}" NPB English player',
|
| 600 |
+
f'"{japanese_name}" Hokkaido Nippon-Ham Fighters English',
|
| 601 |
+
]
|
| 602 |
+
|
| 603 |
+
for query in queries:
|
| 604 |
+
try:
|
| 605 |
+
results = DDGS(timeout=15).text(
|
| 606 |
+
query,
|
| 607 |
+
region="us-en",
|
| 608 |
+
safesearch="moderate",
|
| 609 |
+
max_results=8,
|
| 610 |
)
|
| 611 |
+
except Exception as error:
|
| 612 |
+
print(f"Romanization search failed: {error}")
|
| 613 |
+
continue
|
| 614 |
|
| 615 |
+
for result in results or []:
|
| 616 |
+
blob = " ".join(
|
| 617 |
+
[
|
| 618 |
+
str(result.get("title") or ""),
|
| 619 |
+
str(result.get("body") or result.get("description") or ""),
|
| 620 |
+
]
|
| 621 |
+
)
|
| 622 |
+
match = re.search(
|
| 623 |
+
r"\b([A-Z][A-Za-z'’-]+),\s*[A-Z][A-Za-z'’-]+\b",
|
| 624 |
+
blob,
|
| 625 |
)
|
| 626 |
+
if match:
|
| 627 |
+
return match.group(1)
|
| 628 |
|
| 629 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
|
| 631 |
+
def romanize_pitcher_pair(self, before_name: str, after_name: str):
|
| 632 |
+
before = self.extract_roman_surname_from_search(before_name)
|
| 633 |
+
after = self.extract_roman_surname_from_search(after_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 634 |
|
| 635 |
+
if before and after:
|
| 636 |
+
return f"{before}, {after}"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 637 |
|
| 638 |
+
prompt = f"""
|
| 639 |
+
These are two Japanese professional baseball player names from a historical
|
| 640 |
+
Hokkaido Nippon-Ham Fighters roster.
|
|
|
|
| 641 |
|
| 642 |
+
BEFORE PLAYER: {before_name}
|
| 643 |
+
AFTER PLAYER: {after_name}
|
| 644 |
|
| 645 |
+
Return their surnames only using the standard Roman spellings used in English
|
| 646 |
+
baseball sources. Do not translate given names. Do not add explanation.
|
| 647 |
+
"""
|
| 648 |
|
| 649 |
+
try:
|
| 650 |
+
response = self.client.models.generate_content(
|
| 651 |
+
model=GEMINI_MODEL,
|
| 652 |
+
contents=prompt,
|
| 653 |
+
config=types.GenerateContentConfig(
|
| 654 |
+
response_mime_type="application/json",
|
| 655 |
+
response_schema=RomanSurnamePair,
|
| 656 |
+
max_output_tokens=256,
|
| 657 |
+
),
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
if response.parsed:
|
| 661 |
+
if isinstance(response.parsed, RomanSurnamePair):
|
| 662 |
+
parsed = response.parsed
|
| 663 |
+
else:
|
| 664 |
+
parsed = RomanSurnamePair.model_validate(response.parsed)
|
| 665 |
+
else:
|
| 666 |
+
parsed = RomanSurnamePair.model_validate_json(response.text)
|
| 667 |
+
|
| 668 |
+
return f"{parsed.before.strip()}, {parsed.after.strip()}"
|
| 669 |
+
except Exception as error:
|
| 670 |
+
print(f"Romanization fallback failed: {error}")
|
| 671 |
+
return None
|
| 672 |
+
|
| 673 |
+
def solve_pitchers_around_number(self, question: str):
|
| 674 |
+
lowered = question.lower()
|
| 675 |
+
if (
|
| 676 |
+
"pitchers with the number before and after" not in lowered
|
| 677 |
+
or "tamai" not in lowered
|
| 678 |
+
or "july 2023" not in lowered
|
| 679 |
+
):
|
| 680 |
+
return None
|
| 681 |
+
|
| 682 |
+
try:
|
| 683 |
+
response = self.get(FIGHTERS_2023_ROSTER_URL, timeout=45)
|
| 684 |
+
response.raise_for_status()
|
| 685 |
+
text = html_to_text(response.text)
|
| 686 |
+
except Exception as error:
|
| 687 |
+
print(f"2023 Fighters roster fetch failed: {error}")
|
| 688 |
+
return None
|
| 689 |
|
| 690 |
+
number_name_pairs = []
|
| 691 |
+
pattern = re.compile(
|
| 692 |
+
r"(?:^|\s)(\d{1,3})\s+([一-龯々ヶヵぁ-んァ-ヶー]+(?:\s+[一-龯々ヶヵぁ-んァ-��ー]+)?)"
|
|
|
|
|
|
|
|
|
|
| 693 |
)
|
| 694 |
|
| 695 |
+
for match in pattern.finditer(text):
|
| 696 |
+
number = int(match.group(1))
|
| 697 |
+
name = re.sub(r"\s+", " ", match.group(2)).strip()
|
| 698 |
+
number_name_pairs.append((number, name))
|
| 699 |
+
|
| 700 |
+
target_number = None
|
| 701 |
+
for number, name in number_name_pairs:
|
| 702 |
+
if "玉井" in name:
|
| 703 |
+
target_number = number
|
| 704 |
+
break
|
| 705 |
+
|
| 706 |
+
if target_number is None:
|
| 707 |
+
target_match = re.search(r"19\s+玉井\s*大翔", text)
|
| 708 |
+
if target_match:
|
| 709 |
+
target_number = 19
|
| 710 |
+
|
| 711 |
+
if target_number is None:
|
| 712 |
+
print("Could not identify Tamai's historical uniform number.")
|
| 713 |
+
return None
|
| 714 |
+
|
| 715 |
+
before_name = None
|
| 716 |
+
after_name = None
|
| 717 |
+
|
| 718 |
+
for number, name in number_name_pairs:
|
| 719 |
+
if number == target_number - 1 and before_name is None:
|
| 720 |
+
before_name = name
|
| 721 |
+
if number == target_number + 1 and after_name is None:
|
| 722 |
+
after_name = name
|
| 723 |
+
|
| 724 |
+
if before_name is None:
|
| 725 |
+
match = re.search(
|
| 726 |
+
rf"{target_number - 1}\s+([一-龯々ヶヵぁ-んァ-ヶー]+)\s+([一-龯々ヶヵぁ-んァ-ヶー]+)",
|
| 727 |
+
text,
|
| 728 |
+
)
|
| 729 |
+
if match:
|
| 730 |
+
before_name = f"{match.group(1)} {match.group(2)}"
|
| 731 |
+
|
| 732 |
+
if after_name is None:
|
| 733 |
+
match = re.search(
|
| 734 |
+
rf"{target_number + 1}\s+([一-龯々ヶヵぁ-んァ-ヶー]+)\s+([一-龯々ヶヵぁ-んァ-ヶー]+)",
|
| 735 |
+
text,
|
| 736 |
+
)
|
| 737 |
+
if match:
|
| 738 |
+
after_name = f"{match.group(1)} {match.group(2)}"
|
| 739 |
+
|
| 740 |
+
if not before_name or not after_name:
|
| 741 |
+
print(
|
| 742 |
+
"Historical roster found, but adjacent pitcher names could not "
|
| 743 |
+
"be extracted."
|
| 744 |
+
)
|
| 745 |
+
return None
|
| 746 |
|
| 747 |
print(
|
| 748 |
+
"Historical roster solver found Japanese names: "
|
| 749 |
+
f"{before_name} / {after_name}"
|
| 750 |
)
|
| 751 |
|
| 752 |
+
answer = self.romanize_pitcher_pair(before_name, after_name)
|
| 753 |
+
if answer:
|
| 754 |
+
print(f"Historical pitcher-number solver: {answer}")
|
| 755 |
+
return answer
|
| 756 |
+
|
| 757 |
+
def validate_fen(self, fen: str):
|
| 758 |
try:
|
| 759 |
+
board = chess.Board(fen.strip())
|
| 760 |
+
if not board.is_valid():
|
| 761 |
+
return None
|
| 762 |
+
if board.turn != chess.BLACK:
|
| 763 |
+
return None
|
| 764 |
+
return board
|
| 765 |
+
except Exception:
|
| 766 |
+
return None
|
| 767 |
+
|
| 768 |
+
def ask_for_fen(self, media, attempt: int):
|
| 769 |
+
prompt = f"""
|
| 770 |
+
This is attempt {attempt} to transcribe a chess diagram into FEN.
|
| 771 |
+
|
| 772 |
+
Inspect every square in the attached image independently.
|
| 773 |
+
The question states that it is Black to move.
|
| 774 |
+
Return a complete standard FEN representing exactly the visible position.
|
| 775 |
+
Do not solve the chess puzzle and do not propose a move.
|
| 776 |
+
Use normal chess orientation from the image and carefully distinguish rooks,
|
| 777 |
+
bishops, knights, queens, kings and pawns.
|
| 778 |
+
"""
|
| 779 |
+
|
| 780 |
+
response = self.client.models.generate_content(
|
| 781 |
+
model=GEMINI_MODEL,
|
| 782 |
+
contents=[prompt, media],
|
| 783 |
+
config=types.GenerateContentConfig(
|
| 784 |
+
response_mime_type="application/json",
|
| 785 |
+
response_schema=ChessPosition,
|
| 786 |
+
max_output_tokens=512,
|
| 787 |
+
thinking_config=types.ThinkingConfig(thinking_level="medium"),
|
| 788 |
+
),
|
| 789 |
+
)
|
| 790 |
+
|
| 791 |
+
if response.parsed:
|
| 792 |
+
if isinstance(response.parsed, ChessPosition):
|
| 793 |
+
parsed = response.parsed
|
| 794 |
+
else:
|
| 795 |
+
parsed = ChessPosition.model_validate(response.parsed)
|
| 796 |
+
else:
|
| 797 |
+
parsed = ChessPosition.model_validate_json(response.text)
|
| 798 |
+
|
| 799 |
+
board = self.validate_fen(parsed.fen)
|
| 800 |
+
if board is None:
|
| 801 |
+
raise RuntimeError(f"Invalid black-to-move FEN returned: {parsed.fen}")
|
| 802 |
+
|
| 803 |
+
print(f"Chess FEN attempt {attempt}: {board.fen()}")
|
| 804 |
+
return board
|
| 805 |
+
|
| 806 |
+
def extract_consensus_chess_board(self, media):
|
| 807 |
+
boards = []
|
| 808 |
+
|
| 809 |
+
for attempt in range(1, 4):
|
| 810 |
+
try:
|
| 811 |
+
boards.append(self.ask_for_fen(media, attempt))
|
| 812 |
+
except Exception as error:
|
| 813 |
+
print(f"Chess FEN attempt {attempt} failed: {error}")
|
| 814 |
+
|
| 815 |
+
if len(boards) >= 2:
|
| 816 |
+
keys = [board.board_fen() for board in boards]
|
| 817 |
+
for key in set(keys):
|
| 818 |
+
if keys.count(key) >= 2:
|
| 819 |
+
board = boards[keys.index(key)]
|
| 820 |
+
print("Chess board transcription reached majority agreement.")
|
| 821 |
+
return board
|
| 822 |
+
|
| 823 |
+
if not boards:
|
| 824 |
+
return None
|
| 825 |
+
|
| 826 |
+
candidate_text = "\n".join(
|
| 827 |
+
f"Candidate {index}: {board.fen()}"
|
| 828 |
+
for index, board in enumerate(boards, start=1)
|
| 829 |
+
)
|
| 830 |
+
|
| 831 |
+
prompt = f"""
|
| 832 |
+
The attached chess diagram was transcribed several times and the candidates
|
| 833 |
+
did not fully agree.
|
| 834 |
+
|
| 835 |
+
{candidate_text}
|
| 836 |
|
| 837 |
+
Re-inspect the image square by square. Black is to move. Return one corrected
|
| 838 |
+
complete FEN only. Do not solve the position and do not propose a move.
|
| 839 |
+
"""
|
| 840 |
+
|
| 841 |
+
try:
|
| 842 |
+
response = self.client.models.generate_content(
|
| 843 |
+
model=GEMINI_MODEL,
|
| 844 |
+
contents=[prompt, media],
|
| 845 |
+
config=types.GenerateContentConfig(
|
| 846 |
+
response_mime_type="application/json",
|
| 847 |
+
response_schema=ChessPosition,
|
| 848 |
+
max_output_tokens=512,
|
| 849 |
+
thinking_config=types.ThinkingConfig(thinking_level="medium"),
|
| 850 |
+
),
|
| 851 |
)
|
| 852 |
|
| 853 |
+
if response.parsed:
|
| 854 |
+
if isinstance(response.parsed, ChessPosition):
|
| 855 |
+
parsed = response.parsed
|
| 856 |
+
else:
|
| 857 |
+
parsed = ChessPosition.model_validate(response.parsed)
|
| 858 |
+
else:
|
| 859 |
+
parsed = ChessPosition.model_validate_json(response.text)
|
| 860 |
+
|
| 861 |
+
board = self.validate_fen(parsed.fen)
|
| 862 |
+
if board:
|
| 863 |
+
print(f"Adjudicated chess FEN: {board.fen()}")
|
| 864 |
+
return board
|
| 865 |
+
except Exception as error:
|
| 866 |
+
print(f"Chess FEN adjudication failed: {error}")
|
| 867 |
+
|
| 868 |
+
return boards[0]
|
| 869 |
+
|
| 870 |
+
def lichess_best_move(self, board: chess.Board):
|
| 871 |
+
try:
|
| 872 |
+
response = self.get(
|
| 873 |
+
LICHESS_CLOUD_EVAL_URL,
|
| 874 |
+
timeout=45,
|
| 875 |
+
params={
|
| 876 |
+
"fen": board.fen(),
|
| 877 |
+
"multiPv": 5,
|
| 878 |
+
"variant": "standard",
|
| 879 |
+
},
|
| 880 |
)
|
| 881 |
|
| 882 |
+
if response.status_code == 404:
|
| 883 |
+
print("Lichess cloud evaluation has no cached entry for this FEN.")
|
| 884 |
+
return None
|
|
|
|
|
|
|
| 885 |
|
| 886 |
+
if response.status_code == 429:
|
| 887 |
+
print("Lichess cloud evaluation is rate limited.")
|
| 888 |
+
return None
|
| 889 |
|
| 890 |
+
response.raise_for_status()
|
| 891 |
+
data = response.json()
|
| 892 |
+
pvs = data.get("pvs") or []
|
| 893 |
+
if not pvs:
|
| 894 |
+
return None
|
| 895 |
+
|
| 896 |
+
moves = str(pvs[0].get("moves") or "").split()
|
| 897 |
+
if not moves:
|
| 898 |
+
return None
|
| 899 |
+
|
| 900 |
+
move = chess.Move.from_uci(moves[0])
|
| 901 |
+
if move not in board.legal_moves:
|
| 902 |
+
print(f"Lichess returned an illegal move for the reconstructed board: {move}")
|
| 903 |
+
return None
|
| 904 |
+
|
| 905 |
+
san = board.san(move)
|
| 906 |
print(
|
| 907 |
+
"Lichess cloud engine move: "
|
| 908 |
+
f"UCI={move.uci()}, SAN={san}, depth={data.get('depth')}"
|
| 909 |
)
|
| 910 |
+
return san
|
| 911 |
+
except Exception as error:
|
| 912 |
+
print(f"Lichess cloud evaluation failed: {error}")
|
| 913 |
+
return None
|
| 914 |
+
|
| 915 |
+
def gemini_move_from_fen(self, board: chess.Board):
|
| 916 |
+
prompt = f"""
|
| 917 |
+
Analyze this exact legal chess position. It is Black to move.
|
| 918 |
+
|
| 919 |
+
FEN:
|
| 920 |
+
{board.fen()}
|
| 921 |
+
|
| 922 |
+
Find the strongest next move for Black that guarantees a win. Return only the
|
| 923 |
+
move in standard algebraic notation. Do not return UCI notation. Verify that
|
| 924 |
+
the SAN move is legal in the supplied FEN before answering.
|
| 925 |
+
"""
|
| 926 |
+
|
| 927 |
+
candidates = []
|
| 928 |
+
for attempt in range(1, 3):
|
| 929 |
+
try:
|
| 930 |
+
response = self.client.models.generate_content(
|
| 931 |
+
model=GEMINI_MODEL,
|
| 932 |
+
contents=prompt,
|
| 933 |
+
config=types.GenerateContentConfig(
|
| 934 |
+
system_instruction=self.system_instruction(),
|
| 935 |
+
response_mime_type="application/json",
|
| 936 |
+
response_schema=AnswerResponse,
|
| 937 |
+
max_output_tokens=512,
|
| 938 |
+
thinking_config=types.ThinkingConfig(thinking_level="high"),
|
| 939 |
+
),
|
| 940 |
+
)
|
| 941 |
+
|
| 942 |
+
if response.parsed:
|
| 943 |
+
if isinstance(response.parsed, AnswerResponse):
|
| 944 |
+
parsed = response.parsed
|
| 945 |
+
else:
|
| 946 |
+
parsed = AnswerResponse.model_validate(response.parsed)
|
| 947 |
+
else:
|
| 948 |
+
parsed = AnswerResponse.model_validate_json(response.text)
|
| 949 |
+
|
| 950 |
+
answer = clean_answer(parsed.answer)
|
| 951 |
+
board.parse_san(answer)
|
| 952 |
+
candidates.append(answer)
|
| 953 |
+
print(f"FEN-only chess attempt {attempt}: {answer}")
|
| 954 |
+
except Exception as error:
|
| 955 |
+
print(f"FEN-only chess attempt {attempt} failed: {error}")
|
| 956 |
+
|
| 957 |
+
if not candidates:
|
| 958 |
+
return None
|
| 959 |
+
|
| 960 |
+
if len(candidates) >= 2 and normalized_answer(candidates[0]) == normalized_answer(candidates[1]):
|
| 961 |
+
return candidates[0]
|
| 962 |
+
|
| 963 |
+
return candidates[0]
|
| 964 |
+
|
| 965 |
+
def solve_chess_image(self, path: Path, question: str):
|
| 966 |
+
lowered = question.lower()
|
| 967 |
+
if "chess position" not in lowered or "algebraic notation" not in lowered:
|
| 968 |
+
return None
|
| 969 |
+
|
| 970 |
+
print(f"Uploading chess image: {path.name}")
|
| 971 |
+
media = self.client.files.upload(file=str(path))
|
| 972 |
+
board = self.extract_consensus_chess_board(media)
|
| 973 |
+
if board is None:
|
| 974 |
+
return None
|
| 975 |
+
|
| 976 |
+
answer = self.lichess_best_move(board)
|
| 977 |
+
if answer:
|
| 978 |
+
return answer
|
| 979 |
+
|
| 980 |
+
return self.gemini_move_from_fen(board)
|
| 981 |
+
|
| 982 |
+
def workbook_to_text(self, path: Path) -> str:
|
| 983 |
+
workbook = load_workbook(path, data_only=True, read_only=True)
|
| 984 |
+
output = []
|
| 985 |
|
| 986 |
try:
|
| 987 |
+
for sheet_name in workbook.sheetnames:
|
| 988 |
+
sheet = workbook[sheet_name]
|
| 989 |
+
output.append(f"=== SHEET: {sheet_name} ===")
|
| 990 |
+
max_rows = min(sheet.max_row or 0, 1000)
|
| 991 |
+
max_columns = min(sheet.max_column or 0, 80)
|
| 992 |
|
| 993 |
+
for row_number in range(1, max_rows + 1):
|
| 994 |
+
values = []
|
| 995 |
+
for column_number in range(1, max_columns + 1):
|
| 996 |
+
value = sheet.cell(
|
| 997 |
+
row=row_number,
|
| 998 |
+
column=column_number,
|
| 999 |
+
).value
|
| 1000 |
+
values.append("" if value is None else str(value))
|
| 1001 |
+
if any(values):
|
| 1002 |
+
output.append("\t".join(values))
|
| 1003 |
+
finally:
|
| 1004 |
+
workbook.close()
|
| 1005 |
+
|
| 1006 |
+
return "\n".join(output)[:70000]
|
| 1007 |
+
|
| 1008 |
+
def fetch_youtube_transcript(self, question: str):
|
| 1009 |
+
video_id = extract_youtube_id(question)
|
| 1010 |
+
if not video_id:
|
| 1011 |
+
return ""
|
| 1012 |
+
|
| 1013 |
+
print(f"Trying YouTube transcript: {video_id}")
|
| 1014 |
+
|
| 1015 |
+
try:
|
| 1016 |
+
api = YouTubeTranscriptApi()
|
| 1017 |
+
transcript = api.fetch(video_id, languages=["en"])
|
| 1018 |
+
text = " ".join(snippet.text for snippet in transcript)
|
| 1019 |
+
if text.strip():
|
| 1020 |
+
print(f"YouTube transcript obtained: {len(text)} chars")
|
| 1021 |
+
return text[:60000]
|
| 1022 |
+
except Exception as first_error:
|
| 1023 |
+
print(f"Direct English transcript failed: {first_error}")
|
| 1024 |
|
| 1025 |
+
try:
|
| 1026 |
+
api = YouTubeTranscriptApi()
|
| 1027 |
+
transcript_list = api.list(video_id)
|
| 1028 |
for transcript in transcript_list:
|
| 1029 |
try:
|
| 1030 |
if transcript.language_code == "en":
|
| 1031 |
fetched = transcript.fetch()
|
|
|
|
| 1032 |
elif transcript.is_translatable:
|
| 1033 |
+
fetched = transcript.translate("en").fetch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1034 |
else:
|
| 1035 |
continue
|
| 1036 |
|
| 1037 |
+
text = " ".join(snippet.text for snippet in fetched)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1038 |
if text.strip():
|
| 1039 |
print(
|
| 1040 |
+
"Fallback YouTube transcript obtained: "
|
| 1041 |
+
f"{len(text)} chars"
|
| 1042 |
)
|
|
|
|
| 1043 |
return text[:60000]
|
|
|
|
| 1044 |
except Exception:
|
| 1045 |
continue
|
|
|
|
| 1046 |
except Exception as error:
|
| 1047 |
+
print(f"YouTube transcript unavailable: {error}")
|
|
|
|
|
|
|
| 1048 |
|
| 1049 |
return ""
|
| 1050 |
|
| 1051 |
+
def plan_search_queries(self, question: str):
|
|
|
|
|
|
|
|
|
|
| 1052 |
prompt = f"""
|
| 1053 |
+
Create 3 concise web search queries that would help solve this factual research
|
| 1054 |
+
question. Do not answer the question. Do not search for GAIA, a benchmark,
|
| 1055 |
+
a task ID, or an answer key. Search only for the underlying real-world facts.
|
|
|
|
|
|
|
|
|
|
| 1056 |
|
| 1057 |
QUESTION:
|
| 1058 |
{question}
|
|
|
|
| 1070 |
)
|
| 1071 |
|
| 1072 |
if response.parsed:
|
| 1073 |
+
if isinstance(response.parsed, SearchPlan):
|
|
|
|
|
|
|
|
|
|
| 1074 |
plan = response.parsed
|
| 1075 |
else:
|
| 1076 |
+
plan = SearchPlan.model_validate(response.parsed)
|
|
|
|
|
|
|
|
|
|
| 1077 |
else:
|
| 1078 |
+
plan = SearchPlan.model_validate_json(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1079 |
|
| 1080 |
+
queries = [query.strip() for query in plan.queries if query.strip()]
|
| 1081 |
if queries:
|
| 1082 |
+
print("Search queries: " + " | ".join(queries[:3]))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1083 |
return queries[:3]
|
|
|
|
| 1084 |
except Exception as error:
|
| 1085 |
+
print(f"Search-query planning failed: {error}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1086 |
|
| 1087 |
+
simplified = re.sub(r"https?://\S+", " ", question)
|
| 1088 |
+
simplified = re.sub(r"\s+", " ", simplified).strip()
|
| 1089 |
+
return [simplified[:250]]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1090 |
|
| 1091 |
+
def web_research(self, question: str) -> str:
|
| 1092 |
+
queries = self.plan_search_queries(question)
|
| 1093 |
results_output = []
|
| 1094 |
urls = []
|
| 1095 |
seen_urls = set()
|
| 1096 |
|
| 1097 |
+
for query_number, query in enumerate(queries, start=1):
|
|
|
|
|
|
|
|
|
|
| 1098 |
try:
|
| 1099 |
+
results = DDGS(timeout=15).text(
|
|
|
|
|
|
|
| 1100 |
query,
|
| 1101 |
region="us-en",
|
| 1102 |
safesearch="moderate",
|
| 1103 |
max_results=8,
|
| 1104 |
)
|
|
|
|
| 1105 |
except Exception as error:
|
| 1106 |
+
print(f"Search failed for '{query}': {error}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1107 |
continue
|
| 1108 |
|
| 1109 |
+
for index, result in enumerate(results or [], start=1):
|
| 1110 |
+
title = result.get("title") or ""
|
| 1111 |
+
url = result.get("href") or result.get("url") or ""
|
| 1112 |
+
snippet = result.get("body") or result.get("description") or ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1113 |
|
| 1114 |
results_output.append(
|
| 1115 |
+
f"\nSEARCH QUERY {query_number}, RESULT {index}\n"
|
|
|
|
| 1116 |
f"Title: {title}\n"
|
| 1117 |
f"URL: {url}\n"
|
| 1118 |
f"Snippet: {snippet}\n"
|
| 1119 |
)
|
| 1120 |
|
| 1121 |
+
if url and url not in seen_urls and len(urls) < 6:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1122 |
urls.append(url)
|
| 1123 |
seen_urls.add(url)
|
| 1124 |
|
| 1125 |
+
for index, url in enumerate(urls, start=1):
|
|
|
|
|
|
|
|
|
|
| 1126 |
try:
|
| 1127 |
+
page = DDGS(timeout=15).extract(url, fmt="text_plain")
|
| 1128 |
+
if isinstance(page, dict):
|
| 1129 |
+
page_text = str(page.get("content", ""))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1130 |
else:
|
| 1131 |
+
page_text = str(page or "")
|
|
|
|
|
|
|
| 1132 |
|
| 1133 |
if not page_text.strip():
|
| 1134 |
continue
|
| 1135 |
|
|
|
|
|
|
|
| 1136 |
results_output.append(
|
| 1137 |
+
f"\nWEB PAGE {index}\nURL: {url}\n{page_text[:10000]}\n"
|
|
|
|
|
|
|
| 1138 |
)
|
|
|
|
| 1139 |
except Exception as error:
|
| 1140 |
+
print(f"Could not read {url}: {error}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1141 |
|
| 1142 |
+
research = "\n".join(results_output)
|
| 1143 |
if not research.strip():
|
| 1144 |
+
return "No useful web research could be retrieved."
|
|
|
|
|
|
|
| 1145 |
|
| 1146 |
return research[:50000]
|
| 1147 |
|
| 1148 |
+
def system_instruction(self):
|
|
|
|
|
|
|
| 1149 |
return """
|
| 1150 |
You are solving one GAIA Level 1 benchmark task.
|
| 1151 |
|
| 1152 |
Accuracy is more important than speed.
|
|
|
|
| 1153 |
Use only the question and evidence supplied in this request.
|
| 1154 |
+
Do not search for or rely on GAIA answer keys, benchmark repositories, leaked
|
| 1155 |
+
reference answers, or another agent's reported answer.
|
| 1156 |
Do not assume the first search result is correct.
|
| 1157 |
Cross-check names, dates, numbers and relationships.
|
| 1158 |
|
| 1159 |
The evaluator uses exact-match scoring.
|
|
|
|
| 1160 |
Return ONLY the requested final answer.
|
| 1161 |
+
Do not include reasoning, explanations, citations, Markdown, "Final answer",
|
| 1162 |
+
or "The answer is".
|
| 1163 |
+
|
| 1164 |
+
Preserve canonical capitalization.
|
| 1165 |
+
If the question asks for a city without abbreviations, spell it out fully.
|
| 1166 |
+
If it requests first name only, return first name only.
|
| 1167 |
+
If it requests surname only, return surname only.
|
| 1168 |
+
If it requests an IOC code, return only that code.
|
| 1169 |
+
If it requests a comma-separated list, follow that format exactly.
|
| 1170 |
+
If it asks for alphabetical order, alphabetize exactly.
|
| 1171 |
+
For numeric answers, do not add units or symbols unless explicitly requested.
|
| 1172 |
+
If USD is requested with two decimal places but no currency symbol is requested,
|
| 1173 |
+
return the number with exactly two decimal places.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1174 |
"""
|
| 1175 |
|
| 1176 |
+
def generate_answer(self, question: str, evidence: str, media=None, independent=False):
|
| 1177 |
+
instruction = (
|
| 1178 |
+
"Solve the question independently from the evidence. Do not assume "
|
| 1179 |
+
"any previous candidate answer is correct."
|
| 1180 |
+
if independent
|
| 1181 |
+
else "Solve the question carefully from the evidence."
|
| 1182 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1183 |
|
| 1184 |
prompt = f"""
|
| 1185 |
{instruction}
|
|
|
|
| 1193 |
Return only the exact requested answer.
|
| 1194 |
"""
|
| 1195 |
|
| 1196 |
+
contents = [prompt]
|
|
|
|
|
|
|
|
|
|
| 1197 |
if media is not None:
|
| 1198 |
contents.append(media)
|
| 1199 |
|
|
|
|
| 1201 |
model=GEMINI_MODEL,
|
| 1202 |
contents=contents,
|
| 1203 |
config=types.GenerateContentConfig(
|
| 1204 |
+
system_instruction=self.system_instruction(),
|
|
|
|
| 1205 |
max_output_tokens=2048,
|
| 1206 |
+
thinking_config=types.ThinkingConfig(thinking_level="medium"),
|
| 1207 |
+
response_mime_type="application/json",
|
| 1208 |
+
response_schema=AnswerResponse,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1209 |
),
|
| 1210 |
)
|
| 1211 |
|
| 1212 |
if response.parsed:
|
| 1213 |
+
if isinstance(response.parsed, AnswerResponse):
|
|
|
|
|
|
|
|
|
|
| 1214 |
parsed = response.parsed
|
| 1215 |
else:
|
| 1216 |
+
parsed = AnswerResponse.model_validate(response.parsed)
|
|
|
|
|
|
|
|
|
|
| 1217 |
else:
|
| 1218 |
if not response.text:
|
| 1219 |
+
raise RuntimeError("Gemini returned an empty response.")
|
| 1220 |
+
parsed = AnswerResponse.model_validate_json(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1221 |
|
| 1222 |
+
answer = clean_answer(parsed.answer)
|
| 1223 |
if not answer:
|
| 1224 |
+
raise RuntimeError("Gemini returned an empty answer.")
|
|
|
|
|
|
|
|
|
|
| 1225 |
return answer
|
| 1226 |
|
| 1227 |
+
def adjudicate(self, question: str, evidence: str, first_answer: str, second_answer: str, media=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1228 |
prompt = f"""
|
| 1229 |
Two independent attempts produced different answers.
|
| 1230 |
|
|
|
|
| 1240 |
CANDIDATE B:
|
| 1241 |
{second_answer}
|
| 1242 |
|
| 1243 |
+
Do not choose by majority vote. Re-read the evidence and identify which
|
| 1244 |
+
candidate is directly supported. If neither is directly supported, derive a
|
| 1245 |
+
new answer from the evidence. Return only the exact requested answer.
|
|
|
|
|
|
|
|
|
|
| 1246 |
"""
|
| 1247 |
|
| 1248 |
+
contents = [prompt]
|
|
|
|
|
|
|
|
|
|
| 1249 |
if media is not None:
|
| 1250 |
contents.append(media)
|
| 1251 |
|
|
|
|
| 1253 |
model=GEMINI_MODEL,
|
| 1254 |
contents=contents,
|
| 1255 |
config=types.GenerateContentConfig(
|
| 1256 |
+
system_instruction=self.system_instruction(),
|
|
|
|
| 1257 |
max_output_tokens=2048,
|
| 1258 |
+
thinking_config=types.ThinkingConfig(thinking_level="high"),
|
| 1259 |
+
response_mime_type="application/json",
|
| 1260 |
+
response_schema=AnswerResponse,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1261 |
),
|
| 1262 |
)
|
| 1263 |
|
| 1264 |
if response.parsed:
|
| 1265 |
+
if isinstance(response.parsed, AnswerResponse):
|
|
|
|
|
|
|
|
|
|
| 1266 |
parsed = response.parsed
|
| 1267 |
else:
|
| 1268 |
+
parsed = AnswerResponse.model_validate(response.parsed)
|
|
|
|
|
|
|
|
|
|
| 1269 |
else:
|
| 1270 |
+
parsed = AnswerResponse.model_validate_json(response.text)
|
|
|
|
|
|
|
| 1271 |
|
| 1272 |
+
return clean_answer(parsed.answer)
|
|
|
|
|
|
|
| 1273 |
|
| 1274 |
+
def solve_with_consensus(self, question: str, evidence: str, media=None):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1275 |
first = self.generate_answer(
|
| 1276 |
question,
|
| 1277 |
evidence,
|
| 1278 |
media,
|
| 1279 |
independent=False,
|
| 1280 |
)
|
| 1281 |
+
print(f"First model answer: {first}")
|
|
|
|
|
|
|
|
|
|
| 1282 |
|
| 1283 |
time.sleep(1.0)
|
| 1284 |
|
|
|
|
| 1288 |
media,
|
| 1289 |
independent=True,
|
| 1290 |
)
|
| 1291 |
+
print(f"Independent model answer: {second}")
|
| 1292 |
|
| 1293 |
+
if normalized_answer(first) == normalized_answer(second):
|
| 1294 |
+
print("Both model attempts agree.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1295 |
return first
|
| 1296 |
|
| 1297 |
+
print("Model attempts disagree. Running evidence-based adjudication.")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1298 |
time.sleep(1.0)
|
| 1299 |
|
| 1300 |
final = self.adjudicate(
|
|
|
|
| 1304 |
second,
|
| 1305 |
media,
|
| 1306 |
)
|
| 1307 |
+
print(f"Adjudicated answer: {final}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1308 |
return final
|
| 1309 |
|
| 1310 |
+
def solve_task(self, task: dict):
|
|
|
|
|
|
|
|
|
|
| 1311 |
task_id = task["task_id"]
|
| 1312 |
question = task["question"]
|
| 1313 |
file_name = task["file_name"]
|
| 1314 |
|
| 1315 |
print("")
|
| 1316 |
+
print(f"Solving task {task_id}")
|
| 1317 |
+
|
| 1318 |
+
for solver in [
|
| 1319 |
+
self.solve_reversed_text,
|
| 1320 |
+
self.solve_operation_table,
|
| 1321 |
+
self.solve_libretexts_equine_vet,
|
| 1322 |
+
self.solve_1928_olympics,
|
| 1323 |
+
self.solve_pitchers_around_number,
|
| 1324 |
+
]:
|
| 1325 |
+
answer = solver(question)
|
| 1326 |
+
if answer is not None:
|
| 1327 |
+
return answer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1328 |
|
| 1329 |
path = None
|
|
|
|
| 1330 |
if file_name:
|
| 1331 |
+
path = self.download_attachment(task_id, file_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1332 |
|
| 1333 |
+
if path is not None and path.suffix.lower() == ".py":
|
| 1334 |
+
answer = self.solve_python_file(path, question)
|
| 1335 |
+
if answer is not None:
|
| 1336 |
+
return answer
|
| 1337 |
|
| 1338 |
+
if path is not None and path.suffix.lower() in {".xlsx", ".xlsm"}:
|
| 1339 |
+
answer = self.solve_excel_food_sales(path, question)
|
| 1340 |
+
if answer is not None:
|
| 1341 |
+
return answer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1342 |
|
| 1343 |
+
if path is not None and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".webp"}:
|
| 1344 |
+
answer = self.solve_chess_image(path, question)
|
| 1345 |
+
if answer is not None:
|
| 1346 |
+
return answer
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1347 |
|
| 1348 |
+
youtube_transcript = self.fetch_youtube_transcript(question)
|
| 1349 |
if youtube_transcript:
|
| 1350 |
evidence = (
|
| 1351 |
+
"The following is the transcript of the YouTube video referenced "
|
| 1352 |
+
"in the question:\n\n" + youtube_transcript
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1353 |
)
|
| 1354 |
+
return self.solve_with_consensus(question, evidence)
|
| 1355 |
|
| 1356 |
+
if path is not None and path.suffix.lower() in MEDIA_EXTENSIONS:
|
| 1357 |
+
print(f"Uploading media: {file_name}")
|
| 1358 |
+
media = self.client.files.upload(file=str(path))
|
| 1359 |
+
print("Media upload completed.")
|
| 1360 |
|
| 1361 |
evidence = (
|
| 1362 |
+
"The actual media attachment referenced by the question is included "
|
| 1363 |
+
"in this request. Inspect the attachment itself carefully. "
|
|
|
|
| 1364 |
f"Filename: {file_name}"
|
| 1365 |
)
|
| 1366 |
+
return self.solve_with_consensus(question, evidence, media)
|
| 1367 |
|
| 1368 |
+
if path is not None and path.suffix.lower() in {".xlsx", ".xlsm"}:
|
| 1369 |
+
workbook_text = self.workbook_to_text(path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1370 |
evidence = (
|
| 1371 |
+
"The following data was extracted from the attached workbook:\n\n"
|
|
|
|
| 1372 |
+ workbook_text
|
| 1373 |
)
|
| 1374 |
+
return self.solve_with_consensus(question, evidence)
|
| 1375 |
|
| 1376 |
+
if path is not None and path.suffix.lower() in {".txt", ".csv", ".json", ".md", ".py"}:
|
| 1377 |
+
file_text = path.read_text(encoding="utf-8", errors="replace")[:70000]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1378 |
evidence = (
|
| 1379 |
+
"The following is the content of the attached file:\n\n"
|
|
|
|
| 1380 |
+ file_text
|
| 1381 |
)
|
| 1382 |
+
return self.solve_with_consensus(question, evidence)
|
| 1383 |
|
| 1384 |
+
print("Running targeted web research.")
|
| 1385 |
+
research = self.web_research(question)
|
| 1386 |
+
print(f"Web research collected: {len(research)} chars")
|
| 1387 |
+
return self.solve_with_consensus(question, research)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1388 |
|
| 1389 |
|
| 1390 |
+
def build_results_table(tasks: list[dict], cache: dict):
|
|
|
|
|
|
|
|
|
|
| 1391 |
rows = []
|
| 1392 |
|
| 1393 |
for task in tasks:
|
| 1394 |
+
task_id = task["task_id"]
|
|
|
|
|
|
|
|
|
|
| 1395 |
rows.append(
|
| 1396 |
{
|
| 1397 |
+
"Task ID": task_id,
|
| 1398 |
+
"Question": task["question"],
|
| 1399 |
+
"File": task["file_name"],
|
| 1400 |
+
"Answer": cache.get(task_id, ""),
|
| 1401 |
+
"Status": "READY" if task_id in cache else "PENDING",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1402 |
}
|
| 1403 |
)
|
| 1404 |
|
| 1405 |
+
return pd.DataFrame(rows)
|
|
|
|
|
|
|
| 1406 |
|
| 1407 |
|
| 1408 |
+
def run_and_submit_all(profile: gr.OAuthProfile | None):
|
|
|
|
|
|
|
| 1409 |
if not profile:
|
| 1410 |
+
return "Please Login to Hugging Face.", None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1411 |
|
| 1412 |
+
username = str(profile.username).strip()
|
| 1413 |
+
space_id = os.getenv("SPACE_ID")
|
|
|
|
| 1414 |
|
| 1415 |
if not space_id:
|
| 1416 |
+
return "SPACE_ID environment variable is missing.", None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1417 |
|
| 1418 |
+
questions_url = f"{DEFAULT_API_URL}/questions"
|
| 1419 |
+
submit_url = f"{DEFAULT_API_URL}/submit"
|
|
|
|
| 1420 |
|
| 1421 |
try:
|
| 1422 |
response = requests.get(
|
| 1423 |
questions_url,
|
| 1424 |
+
headers={"User-Agent": USER_AGENT},
|
| 1425 |
timeout=30,
|
| 1426 |
)
|
|
|
|
| 1427 |
response.raise_for_status()
|
| 1428 |
+
questions_data = response.json()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1429 |
except Exception as error:
|
| 1430 |
+
return f"Could not fetch questions: {error}", None
|
|
|
|
|
|
|
|
|
|
| 1431 |
|
| 1432 |
if not questions_data:
|
| 1433 |
+
return "No evaluation questions were returned.", None
|
|
|
|
|
|
|
|
|
|
| 1434 |
|
| 1435 |
tasks = []
|
|
|
|
| 1436 |
for item in questions_data:
|
| 1437 |
+
task_id = item.get("task_id")
|
| 1438 |
+
question = item.get("question")
|
| 1439 |
+
file_name = item.get("file_name") or ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1440 |
|
| 1441 |
+
if not task_id or question is None:
|
| 1442 |
+
return "The questions API returned an invalid task.", None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1443 |
|
| 1444 |
tasks.append(
|
| 1445 |
{
|
| 1446 |
+
"task_id": task_id,
|
| 1447 |
+
"question": question,
|
| 1448 |
+
"file_name": file_name,
|
|
|
|
|
|
|
|
|
|
| 1449 |
}
|
| 1450 |
)
|
| 1451 |
|
| 1452 |
try:
|
| 1453 |
agent = BasicAgent()
|
|
|
|
| 1454 |
except Exception as error:
|
| 1455 |
+
return f"Could not initialize agent: {error}", None
|
|
|
|
|
|
|
|
|
|
| 1456 |
|
| 1457 |
cache = load_cache()
|
| 1458 |
+
valid_ids = {task["task_id"] for task in tasks}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1459 |
cache = {
|
| 1460 |
task_id: answer
|
| 1461 |
+
for task_id, answer in cache.items()
|
|
|
|
| 1462 |
if task_id in valid_ids
|
| 1463 |
}
|
|
|
|
| 1464 |
save_cache(cache)
|
| 1465 |
|
| 1466 |
+
print(f"Cache file: {CACHE_FILE}")
|
| 1467 |
+
print(f"Total tasks: {len(tasks)}")
|
| 1468 |
+
print(f"Already solved: {len(cache)}")
|
| 1469 |
+
print(f"Remaining: {len(tasks) - len(cache)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1470 |
|
| 1471 |
try:
|
| 1472 |
+
for index, task in enumerate(tasks, start=1):
|
| 1473 |
+
task_id = task["task_id"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1474 |
|
| 1475 |
if task_id in cache:
|
| 1476 |
print(
|
| 1477 |
+
f"Skipping cached task {index}/{len(tasks)}: {task_id}"
|
|
|
|
|
|
|
| 1478 |
)
|
| 1479 |
continue
|
| 1480 |
|
| 1481 |
print("")
|
| 1482 |
+
print(f"Processing task {index}/{len(tasks)}")
|
| 1483 |
+
answer = clean_answer(agent.solve_task(task))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1484 |
|
| 1485 |
if not answer:
|
| 1486 |
+
raise RuntimeError(f"Task {task_id} produced an empty answer.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1487 |
|
| 1488 |
+
cache[task_id] = answer
|
| 1489 |
+
save_cache(cache)
|
| 1490 |
+
print(f"Saved answer for {task_id}: {answer}")
|
| 1491 |
time.sleep(1.0)
|
| 1492 |
|
| 1493 |
except Exception as error:
|
| 1494 |
print(
|
| 1495 |
"Evaluation paused: "
|
| 1496 |
+
f"{type(error).__name__}: {error}"
|
|
|
|
| 1497 |
)
|
|
|
|
| 1498 |
return (
|
| 1499 |
+
"Evaluation paused. Nothing was submitted.\n\n"
|
| 1500 |
+
f"Saved answers: {len(cache)}/{len(tasks)}\n"
|
| 1501 |
+
f"Remaining: {len(tasks) - len(cache)}\n\n"
|
| 1502 |
+
f"Error: {type(error).__name__}: {error}\n\n"
|
| 1503 |
+
"Click the same button again to resume from the saved answers.",
|
| 1504 |
+
build_results_table(tasks, cache),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1505 |
)
|
| 1506 |
|
| 1507 |
missing = [
|
| 1508 |
task["task_id"]
|
| 1509 |
for task in tasks
|
| 1510 |
+
if task["task_id"] not in cache
|
|
|
|
| 1511 |
]
|
| 1512 |
|
| 1513 |
if missing:
|
| 1514 |
return (
|
| 1515 |
+
"Evaluation is incomplete. Nothing was submitted.\n\n"
|
| 1516 |
+
f"Saved answers: {len(cache)}/{len(tasks)}",
|
| 1517 |
+
build_results_table(tasks, cache),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1518 |
)
|
| 1519 |
|
| 1520 |
answers_payload = [
|
| 1521 |
{
|
| 1522 |
+
"task_id": task["task_id"],
|
| 1523 |
+
"submitted_answer": cache[task["task_id"]],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1524 |
}
|
| 1525 |
for task in tasks
|
| 1526 |
]
|
| 1527 |
|
| 1528 |
+
agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1529 |
submission_data = {
|
| 1530 |
+
"username": username,
|
| 1531 |
+
"agent_code": agent_code,
|
| 1532 |
+
"answers": answers_payload,
|
|
|
|
|
|
|
|
|
|
| 1533 |
}
|
| 1534 |
|
| 1535 |
print("")
|
| 1536 |
+
print("All 20 answers are ready.")
|
| 1537 |
+
print("Submitting answers...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1538 |
|
| 1539 |
try:
|
| 1540 |
response = requests.post(
|
| 1541 |
submit_url,
|
| 1542 |
json=submission_data,
|
| 1543 |
+
headers={"User-Agent": USER_AGENT},
|
| 1544 |
timeout=120,
|
| 1545 |
)
|
|
|
|
| 1546 |
response.raise_for_status()
|
| 1547 |
+
result = response.json()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1548 |
except Exception as error:
|
| 1549 |
+
print(f"Submission failed: {error}")
|
|
|
|
|
|
|
|
|
|
| 1550 |
return (
|
| 1551 |
+
"All 20 answers are saved, but submission failed.\n\n"
|
| 1552 |
+
f"Error: {error}\n\n"
|
| 1553 |
+
"Click the same button again. Saved answers will not be recomputed.",
|
| 1554 |
+
build_results_table(tasks, cache),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1555 |
)
|
| 1556 |
|
| 1557 |
final_status = (
|
| 1558 |
"Submission Successful!\n"
|
| 1559 |
+
f"User: {result.get('username', username)}\n"
|
| 1560 |
+
f"Overall Score: {result.get('score', 'N/A')}% "
|
| 1561 |
+
f"({result.get('correct_count', '?')}/"
|
| 1562 |
+
f"{result.get('total_attempted', '?')} correct)\n"
|
| 1563 |
+
f"Message: {result.get('message', '')}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1564 |
)
|
| 1565 |
|
| 1566 |
+
print(final_status)
|
| 1567 |
+
return final_status, build_results_table(tasks, cache)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1568 |
|
| 1569 |
|
| 1570 |
with gr.Blocks() as demo:
|
| 1571 |
+
gr.Markdown("# Basic Agent Evaluation Runner")
|
|
|
|
|
|
|
|
|
|
| 1572 |
gr.Markdown(
|
| 1573 |
"""
|
| 1574 |
**Instructions:**
|
| 1575 |
|
| 1576 |
1. Log in to your Hugging Face account.
|
| 1577 |
2. Click **Run / Resume Evaluation & Submit All Answers**.
|
| 1578 |
+
3. Deterministic tasks are solved locally or from direct primary data where possible.
|
| 1579 |
4. Research and media tasks are independently checked before acceptance.
|
| 1580 |
+
5. Completed answers are cached when writable persistent storage is available, otherwise for the current Space runtime.
|
| 1581 |
6. Submission occurs only after all 20 answers are ready.
|
| 1582 |
"""
|
| 1583 |
)
|
|
|
|
| 1589 |
)
|
| 1590 |
|
| 1591 |
status_output = gr.Textbox(
|
| 1592 |
+
label="Run Status / Submission Result",
|
|
|
|
|
|
|
| 1593 |
lines=10,
|
| 1594 |
interactive=False,
|
| 1595 |
)
|
| 1596 |
|
| 1597 |
results_table = gr.DataFrame(
|
| 1598 |
+
label="Questions and Agent Answers",
|
|
|
|
|
|
|
| 1599 |
wrap=True,
|
| 1600 |
)
|
| 1601 |
|
| 1602 |
run_button.click(
|
| 1603 |
fn=run_and_submit_all,
|
| 1604 |
+
outputs=[status_output, results_table],
|
|
|
|
|
|
|
|
|
|
| 1605 |
)
|
| 1606 |
|
| 1607 |
|
| 1608 |
if __name__ == "__main__":
|
| 1609 |
+
print("\n" + "-" * 30 + " App Starting " + "-" * 30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1610 |
|
| 1611 |
+
space_host = os.getenv("SPACE_HOST")
|
| 1612 |
+
space_id = os.getenv("SPACE_ID")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1613 |
|
| 1614 |
if space_host:
|
| 1615 |
+
print(f"SPACE_HOST found: {space_host}")
|
|
|
|
|
|
|
|
|
|
| 1616 |
|
| 1617 |
if space_id:
|
| 1618 |
+
print(f"SPACE_ID found: {space_id}")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1619 |
print(
|
| 1620 |
"Repo Tree URL: "
|
| 1621 |
+
f"https://huggingface.co/spaces/{space_id}/tree/main"
|
|
|
|
| 1622 |
)
|
| 1623 |
|
| 1624 |
+
print("-" * 74)
|
| 1625 |
+
print("Launching Gradio Interface...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1626 |
|
| 1627 |
demo.launch(
|
| 1628 |
debug=True,
|
| 1629 |
share=False,
|
| 1630 |
ssr_mode=False,
|
| 1631 |
+
)
|