Spaces:
Sleeping
Sleeping
Joachim M. Vis commited on
Upload app.py
Browse files
app.py
CHANGED
|
@@ -31,22 +31,13 @@ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None, progress=gr.Pr
|
|
| 31 |
Tuple containing status message and results dataframe
|
| 32 |
"""
|
| 33 |
space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
|
| 34 |
-
|
| 35 |
-
|
|
|
|
|
|
|
| 36 |
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
if profile:
|
| 40 |
-
username = f"{profile.username}"
|
| 41 |
-
logger.info(f"User logged in: {username}")
|
| 42 |
-
else:
|
| 43 |
-
logger.warning("User not logged in.")
|
| 44 |
-
return "Please Login to Hugging Face with the button.", None
|
| 45 |
-
else:
|
| 46 |
-
# Demo mode when OAuth is not configured
|
| 47 |
-
demo_mode = True
|
| 48 |
-
username = "demo_user"
|
| 49 |
-
logger.info("Running in demonstration mode (OAuth not configured)")
|
| 50 |
|
| 51 |
api_url = DEFAULT_API_URL
|
| 52 |
questions_url = f"{api_url}/questions"
|
|
@@ -147,18 +138,6 @@ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None, progress=gr.Pr
|
|
| 147 |
error_msg = "Agent did not produce any answers to submit."
|
| 148 |
logger.warning(error_msg)
|
| 149 |
return error_msg, pd.DataFrame(results_log)
|
| 150 |
-
|
| 151 |
-
# Skip submission in demo mode
|
| 152 |
-
if demo_mode:
|
| 153 |
-
progress(1.0, desc="Demo run completed")
|
| 154 |
-
status_message = (
|
| 155 |
-
f"✅ Demo run completed with {len(answers_payload)} processed questions.\n\n"
|
| 156 |
-
"Note: This Space is running in demonstration mode without OAuth login. "
|
| 157 |
-
"Results were not submitted to the leaderboard. "
|
| 158 |
-
"To enable submissions, configure OAuth for this Space."
|
| 159 |
-
)
|
| 160 |
-
logger.info("Demo run completed - no submission attempted")
|
| 161 |
-
return status_message, pd.DataFrame(results_log)
|
| 162 |
|
| 163 |
# Only proceed with submission if not in demo mode
|
| 164 |
progress(0.8, desc="Preparing submission...")
|
|
@@ -178,7 +157,7 @@ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None, progress=gr.Pr
|
|
| 178 |
score = result_data.get('score', 'N/A')
|
| 179 |
|
| 180 |
final_status = (
|
| 181 |
-
f"
|
| 182 |
f"User: {result_data.get('username')}\n"
|
| 183 |
f"Overall Score: {score}% "
|
| 184 |
f"({correct_count}/{total_attempted} correct)\n"
|
|
@@ -210,86 +189,63 @@ def run_and_submit_all(profile: Optional[gr.OAuthProfile] = None, progress=gr.Pr
|
|
| 210 |
error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
|
| 211 |
except requests.exceptions.JSONDecodeError:
|
| 212 |
error_detail += f" Response: {e.response.text[:500]}"
|
| 213 |
-
status_message = f"
|
| 214 |
logger.error(status_message)
|
| 215 |
results_df = pd.DataFrame(results_log)
|
| 216 |
return status_message, results_df
|
| 217 |
except requests.exceptions.Timeout:
|
| 218 |
progress(1.0, desc="Submission timed out!")
|
| 219 |
-
status_message = "
|
| 220 |
logger.error(status_message)
|
| 221 |
results_df = pd.DataFrame(results_log)
|
| 222 |
return status_message, results_df
|
| 223 |
except requests.exceptions.RequestException as e:
|
| 224 |
progress(1.0, desc="Network error!")
|
| 225 |
-
status_message = f"
|
| 226 |
logger.error(status_message)
|
| 227 |
results_df = pd.DataFrame(results_log)
|
| 228 |
return status_message, results_df
|
| 229 |
except Exception as e:
|
| 230 |
progress(1.0, desc="Error during submission!")
|
| 231 |
-
status_message = f"
|
| 232 |
logger.error(status_message)
|
| 233 |
results_df = pd.DataFrame(results_log)
|
| 234 |
return status_message, results_df
|
| 235 |
|
| 236 |
|
| 237 |
-
# Check if we're running in a HF Space environment
|
| 238 |
space_host = os.getenv("SPACE_HOST")
|
| 239 |
-
hf_oauth_enabled = False # Default assumption
|
| 240 |
|
|
|
|
| 241 |
try:
|
| 242 |
-
|
| 243 |
-
if space_host:
|
| 244 |
-
from huggingface_hub import HfApi
|
| 245 |
-
if os.getenv("OAUTH_CLIENT_ID"):
|
| 246 |
-
hf_oauth_enabled = True
|
| 247 |
except ImportError:
|
| 248 |
-
logger.warning("huggingface_hub not available -
|
| 249 |
except Exception as e:
|
| 250 |
-
logger.warning(f"Error
|
| 251 |
|
| 252 |
with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
| 253 |
gr.Markdown("# GAIA Agent Evaluation Runner")
|
| 254 |
-
|
| 255 |
with gr.Tabs():
|
| 256 |
with gr.TabItem("Run Evaluation"):
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
"""
|
| 261 |
-
**Instructions:**
|
| 262 |
-
|
| 263 |
-
1. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
|
| 264 |
-
2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run the GAIA agent, submit answers, and see the score.
|
| 265 |
-
3. The progress bar will show you the current status of the evaluation.
|
| 266 |
-
4. Results will appear in the table below once processing is complete.
|
| 267 |
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
gr.Markdown(
|
| 273 |
-
"""
|
| 274 |
-
**Instructions:**
|
| 275 |
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
**Note:** This Space is running in demonstration mode without OAuth login. Submission to the leaderboard is disabled.
|
| 281 |
-
"""
|
| 282 |
-
)
|
| 283 |
|
| 284 |
with gr.Row():
|
| 285 |
with gr.Column(scale=1):
|
| 286 |
-
|
| 287 |
-
if hf_oauth_enabled:
|
| 288 |
-
gr.LoginButton()
|
| 289 |
-
else:
|
| 290 |
-
gr.Markdown("*OAuth login not configured*")
|
| 291 |
with gr.Column(scale=2):
|
| 292 |
-
run_button = gr.Button("Run Evaluation
|
| 293 |
variant="primary", size="lg")
|
| 294 |
|
| 295 |
status_output = gr.Textbox(
|
|
@@ -384,21 +340,15 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
|
| 384 |
outputs=[results_accordion]
|
| 385 |
)
|
| 386 |
|
| 387 |
-
# Add
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
1. Go to the Space settings
|
| 397 |
-
2. Under "Repository secrets", set `hf_oauth: true`
|
| 398 |
-
|
| 399 |
-
Currently running in demonstration mode with submission features disabled.
|
| 400 |
-
"""
|
| 401 |
-
)
|
| 402 |
|
| 403 |
if __name__ == "__main__":
|
| 404 |
space_host_startup = os.getenv("SPACE_HOST")
|
|
@@ -406,8 +356,6 @@ if __name__ == "__main__":
|
|
| 406 |
|
| 407 |
# Handle Space host configurations
|
| 408 |
if space_host_startup:
|
| 409 |
-
if not os.getenv("OAUTH_CLIENT_ID"):
|
| 410 |
-
logger.warning("OAuth not configured for this Space. Running in demonstration mode.")
|
| 411 |
logger.info(f"Running on Hugging Face Space host: {space_host_startup}")
|
| 412 |
|
| 413 |
# Print repository URLs if SPACE_ID is available
|
|
|
|
| 31 |
Tuple containing status message and results dataframe
|
| 32 |
"""
|
| 33 |
space_id = os.getenv("SPACE_ID") # Get the SPACE_ID for sending link to the code
|
| 34 |
+
# OAuth is REQUIRED for assessment submission
|
| 35 |
+
if not profile:
|
| 36 |
+
logger.warning("User not logged in.")
|
| 37 |
+
return "Please Login to Hugging Face with the button. OAuth login is required for assessment submission.", None
|
| 38 |
|
| 39 |
+
username = f"{profile.username}"
|
| 40 |
+
logger.info(f"User logged in: {username}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
api_url = DEFAULT_API_URL
|
| 43 |
questions_url = f"{api_url}/questions"
|
|
|
|
| 138 |
error_msg = "Agent did not produce any answers to submit."
|
| 139 |
logger.warning(error_msg)
|
| 140 |
return error_msg, pd.DataFrame(results_log)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
# Only proceed with submission if not in demo mode
|
| 143 |
progress(0.8, desc="Preparing submission...")
|
|
|
|
| 157 |
score = result_data.get('score', 'N/A')
|
| 158 |
|
| 159 |
final_status = (
|
| 160 |
+
f"[SUCCESS] Submission Successful!\n"
|
| 161 |
f"User: {result_data.get('username')}\n"
|
| 162 |
f"Overall Score: {score}% "
|
| 163 |
f"({correct_count}/{total_attempted} correct)\n"
|
|
|
|
| 189 |
error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
|
| 190 |
except requests.exceptions.JSONDecodeError:
|
| 191 |
error_detail += f" Response: {e.response.text[:500]}"
|
| 192 |
+
status_message = f"[ERROR] Submission Failed: {error_detail}"
|
| 193 |
logger.error(status_message)
|
| 194 |
results_df = pd.DataFrame(results_log)
|
| 195 |
return status_message, results_df
|
| 196 |
except requests.exceptions.Timeout:
|
| 197 |
progress(1.0, desc="Submission timed out!")
|
| 198 |
+
status_message = "[ERROR] Submission Failed: The request timed out."
|
| 199 |
logger.error(status_message)
|
| 200 |
results_df = pd.DataFrame(results_log)
|
| 201 |
return status_message, results_df
|
| 202 |
except requests.exceptions.RequestException as e:
|
| 203 |
progress(1.0, desc="Network error!")
|
| 204 |
+
status_message = f"[ERROR] Submission Failed: Network error - {e}"
|
| 205 |
logger.error(status_message)
|
| 206 |
results_df = pd.DataFrame(results_log)
|
| 207 |
return status_message, results_df
|
| 208 |
except Exception as e:
|
| 209 |
progress(1.0, desc="Error during submission!")
|
| 210 |
+
status_message = f"[ERROR] An unexpected error occurred during submission: {e}"
|
| 211 |
logger.error(status_message)
|
| 212 |
results_df = pd.DataFrame(results_log)
|
| 213 |
return status_message, results_df
|
| 214 |
|
| 215 |
|
| 216 |
+
# Check if we're running in a HF Space environment
|
| 217 |
space_host = os.getenv("SPACE_HOST")
|
|
|
|
| 218 |
|
| 219 |
+
# Import HfApi for OAuth functionality
|
| 220 |
try:
|
| 221 |
+
from huggingface_hub import HfApi
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
except ImportError:
|
| 223 |
+
logger.warning("huggingface_hub not available - please install it for OAuth functionality")
|
| 224 |
except Exception as e:
|
| 225 |
+
logger.warning(f"Error importing huggingface_hub: {e}")
|
| 226 |
|
| 227 |
with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
| 228 |
gr.Markdown("# GAIA Agent Evaluation Runner")
|
|
|
|
| 229 |
with gr.Tabs():
|
| 230 |
with gr.TabItem("Run Evaluation"):
|
| 231 |
+
gr.Markdown(
|
| 232 |
+
"""
|
| 233 |
+
**Instructions:**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 234 |
|
| 235 |
+
1. Log in to your Hugging Face account using the button below. This uses your HF username for submission.
|
| 236 |
+
2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run the GAIA agent, submit answers, and see the score.
|
| 237 |
+
3. The progress bar will show you the current status of the evaluation.
|
| 238 |
+
4. Results will appear in the table below once processing is complete.
|
|
|
|
|
|
|
|
|
|
| 239 |
|
| 240 |
+
**Note:** Running the evaluation may take some time as the agent processes all questions. Please be patient.
|
| 241 |
+
"""
|
| 242 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
|
| 244 |
with gr.Row():
|
| 245 |
with gr.Column(scale=1):
|
| 246 |
+
gr.LoginButton()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
with gr.Column(scale=2):
|
| 248 |
+
run_button = gr.Button("Run Evaluation & Submit All Answers",
|
| 249 |
variant="primary", size="lg")
|
| 250 |
|
| 251 |
status_output = gr.Textbox(
|
|
|
|
| 340 |
outputs=[results_accordion]
|
| 341 |
)
|
| 342 |
|
| 343 |
+
# Add note that OAuth is required
|
| 344 |
+
gr.Markdown(
|
| 345 |
+
"""
|
| 346 |
+
## [WARNING] Important Note
|
| 347 |
+
|
| 348 |
+
This GAIA Assessment requires OAuth login for submission to the leaderboard.
|
| 349 |
+
Please make sure to log in with your Hugging Face account before running the evaluation.
|
| 350 |
+
"""
|
| 351 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 352 |
|
| 353 |
if __name__ == "__main__":
|
| 354 |
space_host_startup = os.getenv("SPACE_HOST")
|
|
|
|
| 356 |
|
| 357 |
# Handle Space host configurations
|
| 358 |
if space_host_startup:
|
|
|
|
|
|
|
| 359 |
logger.info(f"Running on Hugging Face Space host: {space_host_startup}")
|
| 360 |
|
| 361 |
# Print repository URLs if SPACE_ID is available
|