NidhiS09 commited on
Commit
4759b61
Β·
verified Β·
1 Parent(s): 290e58a

Update src/main.py

Browse files
Files changed (1) hide show
  1. src/main.py +418 -153
src/main.py CHANGED
@@ -1,12 +1,10 @@
1
- # import streamlit as st
2
- # st.write("hello")
3
-
4
-
5
  import os
6
  import json
7
  import uuid
8
  import time
9
  import tempfile
 
 
10
  from typing import Any, Dict, List, Tuple
11
 
12
  import streamlit as st
@@ -20,55 +18,155 @@ from huggingface_hub.utils import HfHubHTTPError
20
  # =========================
21
 
22
  st.set_page_config(
23
- page_title="AI Benchmark Arena",
24
  page_icon="πŸ†",
25
  layout="wide",
26
  initial_sidebar_state="expanded",
27
  )
28
 
29
- #Set this to the private dataset repo that acts as the "database"
30
  DB_REPO_ID = os.getenv("DB_REPO_ID", "NidhiS09/VizWiz-submissions-db")
31
  DB_REPO_TYPE = "dataset"
32
- # print(DB_REPO_ID)
33
-
34
-
35
- # This must exist as a Space Secret in the PUBLIC UI Space
36
  SUBMISSIONS_TOKEN = os.getenv("SUBMISSIONS_TOKEN", "")
37
 
38
- # Phase config (copied from EvalAI config intent)
 
39
  PHASES = [
40
- {"label": "Dev (qeury-dev2024)", "codename": "test-dev2024"},
41
  {"label": "Standard (query-standard2024)", "codename": "test-standard2024"},
42
  {"label": "Challenge (query-challenge2024)", "codename": "test-challenge2024"},
43
  ]
44
 
45
  CHALLENGE_TYPES = ["Object Detection", "Instance Segmentation"]
46
 
47
- # Leaderboard columns from your EvalAI yaml
48
  LEADERBOARD_METRICS = ["bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50"]
49
  DEFAULT_SORT_METRIC = "segm_AP50"
50
 
51
  # =========================
52
- # HELPERS
53
  # =========================
54
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
  def _require_token() -> None:
56
  if not SUBMISSIONS_TOKEN:
57
  st.error(
58
- "Missing SUBMISSIONS_TOKEN. Add it in Space Settings β†’ Secrets "
59
- "(token must have read/write access ONLY to the private DB dataset repo)."
60
  )
61
  st.stop()
62
 
63
 
64
  def _validate_submission_json(obj: Any) -> Tuple[bool, str]:
65
- """
66
- Validates the submission format from your instructions:
67
- - Top-level must be a list
68
- - Each item must be a dict containing:
69
- image_id (int), score (number), category_id (int), area (number),
70
- bbox ([x,y,w,h]), segmentation (list)
71
- """
72
  if not isinstance(obj, list):
73
  return False, "Submission must be a JSON list of annotations."
74
 
@@ -77,47 +175,38 @@ def _validate_submission_json(obj: Any) -> Tuple[bool, str]:
77
  for i, ann in enumerate(obj):
78
  if not isinstance(ann, dict):
79
  return False, f"Annotation at index {i} must be an object/dict."
80
-
81
  missing = required_keys - set(ann.keys())
82
  if missing:
83
  return False, f"Annotation at index {i} missing keys: {sorted(list(missing))}"
84
-
85
- # Basic type checks
86
  if not isinstance(ann["image_id"], int):
87
  return False, f"image_id at index {i} must be an integer."
88
  if not isinstance(ann["category_id"], int):
89
  return False, f"category_id at index {i} must be an integer."
90
-
91
  if not isinstance(ann["score"], (int, float)):
92
  return False, f"score at index {i} must be a number."
93
-
94
  if not isinstance(ann["area"], (int, float)):
95
  return False, f"area at index {i} must be a number."
96
-
97
  bbox = ann["bbox"]
98
  if not (isinstance(bbox, list) and len(bbox) == 4 and all(isinstance(x, (int, float)) for x in bbox)):
99
  return False, f"bbox at index {i} must be a list of 4 numbers: [x, y, w, h]."
100
-
101
- segm = ann["segmentation"]
102
- if not isinstance(segm, list):
103
  return False, f"segmentation at index {i} must be a list."
104
 
105
  return True, "OK"
106
 
107
 
108
- def _upload_json(api: HfApi, data: Dict[str, Any] | List[Any], path_in_repo: str) -> None:
109
  with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
110
  json.dump(data, tmp, ensure_ascii=False)
111
  tmp_path = tmp.name
112
-
113
  try:
114
- api.upload_file(
115
  path_or_fileobj=tmp_path,
116
  path_in_repo=path_in_repo,
117
  repo_id=DB_REPO_ID,
118
  repo_type=DB_REPO_TYPE,
119
  token=SUBMISSIONS_TOKEN,
120
- commit_message=f"Add {path_in_repo}",
121
  )
122
  finally:
123
  try:
@@ -134,13 +223,10 @@ def _create_submission_record(
134
  phase_codename: str,
135
  challenge_type: str,
136
  original_filename: str,
 
 
137
  ) -> str:
138
- """
139
- Writes pred/meta/status to the private DB dataset repo.
140
- Returns submission_id.
141
- """
142
  _require_token()
143
- api = HfApi()
144
 
145
  submission_id = str(uuid.uuid4())
146
  ts = int(time.time())
@@ -153,43 +239,34 @@ def _create_submission_record(
153
  "challenge_type": challenge_type,
154
  "timestamp": ts,
155
  "original_filename": original_filename,
 
 
156
  }
157
 
158
  status = {"state": "queued", "timestamp": ts}
159
-
160
  base = f"submissions/{submission_id}"
161
- _upload_json(api, pred, f"{base}/pred.json")
162
- _upload_json(api, meta, f"{base}/meta.json")
163
- _upload_json(api, status, f"{base}/status.json")
 
164
 
165
  return submission_id
166
 
167
 
168
- def _download_leaderboard_jsonl() -> str | None:
169
- """
170
- Downloads leaderboard.jsonl from the DB repo.
171
- Returns local path or None if missing.
172
- """
173
  _require_token()
174
  try:
175
- return hf_hub_download(
176
  repo_id=DB_REPO_ID,
177
  repo_type=DB_REPO_TYPE,
178
  filename="leaderboard.jsonl",
179
  token=SUBMISSIONS_TOKEN,
180
  )
181
  except HfHubHTTPError as e:
182
- # Most common: 404 when file doesn't exist yet
183
  if "404" in str(e):
184
- return None
185
  raise
186
 
187
-
188
- def _load_leaderboard_df() -> pd.DataFrame:
189
- path = _download_leaderboard_jsonl()
190
- if path is None:
191
- return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
192
-
193
  rows = []
194
  with open(path, "r", encoding="utf-8") as f:
195
  for line in f:
@@ -199,33 +276,195 @@ def _load_leaderboard_df() -> pd.DataFrame:
199
  try:
200
  rows.append(json.loads(line))
201
  except json.JSONDecodeError:
202
- # Skip malformed lines rather than crashing the UI
203
  continue
204
 
205
  if not rows:
206
  return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
207
 
208
  df = pd.DataFrame(rows)
209
-
210
- # Ensure columns exist
211
  for col in ["team", "model", "phase_codename", "timestamp", *LEADERBOARD_METRICS]:
212
  if col not in df.columns:
213
  df[col] = None
214
 
215
- # Sort descending by default metric
216
  if DEFAULT_SORT_METRIC in df.columns:
217
  df = df.sort_values(by=DEFAULT_SORT_METRIC, ascending=False, kind="mergesort")
218
 
219
  return df
220
 
221
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  # =========================
223
- # UI
224
  # =========================
225
- #2 FUNCTIONS WERE HERE, RENDER_OVERVIEW AND RENDER EVAL
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
 
227
  def page_submit():
228
- st.header("πŸš€ Submit your Predictions")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
229
 
230
  col1, col2 = st.columns([2, 1])
231
 
@@ -233,26 +472,21 @@ def page_submit():
233
  st.subheader("Submission Info")
234
  team = st.text_input("Team / Display Name", value=st.session_state.get("team", ""))
235
  model_name = st.text_input("Model Name", value=st.session_state.get("model_name", ""))
236
-
237
  phase_label = st.selectbox("Phase", [p["label"] for p in PHASES])
238
  phase_codename = next(p["codename"] for p in PHASES if p["label"] == phase_label)
239
-
240
  challenge_type = st.radio("Challenge type", CHALLENGE_TYPES, horizontal=False)
241
-
242
  st.session_state["team"] = team
243
  st.session_state["model_name"] = model_name
244
-
245
- st.caption("Your submission will be queued for evaluation. Scores appear on the leaderboard after processing.")
246
 
247
  with col1:
248
  st.subheader("Upload Submission File")
249
  uploaded_file = st.file_uploader("Choose a JSON file", type=["json"])
250
 
251
  if uploaded_file is None:
252
- st.info("Upload a JSON file that contains a list of annotations.")
253
  return
254
 
255
- # Parse JSON
256
  try:
257
  raw = uploaded_file.getvalue().decode("utf-8")
258
  pred_obj = json.loads(raw)
@@ -271,13 +505,19 @@ def page_submit():
271
 
272
  if submit_clicked:
273
  if not team.strip():
274
- st.error("Please enter Team / Display Name.")
275
  return
276
  if not model_name.strip():
277
- st.error("Please enter Model Name.")
 
 
 
 
 
 
278
  return
279
 
280
- with st.spinner("Uploading submission to the private database repo..."):
281
  try:
282
  submission_id = _create_submission_record(
283
  pred=pred_obj,
@@ -286,6 +526,8 @@ def page_submit():
286
  phase_codename=phase_codename,
287
  challenge_type=challenge_type,
288
  original_filename=uploaded_file.name,
 
 
289
  )
290
  except Exception as e:
291
  st.error(f"Upload failed: {e}")
@@ -294,106 +536,129 @@ def page_submit():
294
  st.balloons()
295
  st.success("Submission queued successfully!")
296
  st.code(f"Submission ID: {submission_id}")
 
297
 
298
- st.info("Next: the private evaluator will score your submission and update the leaderboard.")
299
 
 
 
 
300
 
301
- def page_leaderboard():
302
- st.header("πŸ† Leaderboard")
303
- st.write(f"Ranked by **{DEFAULT_SORT_METRIC}** (descending).")
304
 
305
- with st.spinner("Loading leaderboard from private database repo..."):
306
- try:
307
- df = _load_leaderboard_df()
308
- except Exception as e:
309
- st.error(f"Could not load leaderboard: {e}")
310
- return
311
 
312
- if df.empty:
313
- st.info("No scored submissions yet. Submit a model to get started!")
314
  return
315
 
316
- # Add Rank column
317
- df_display = df.copy()
318
- df_display.insert(0, "Rank", range(1, len(df_display) + 1))
319
 
320
- # Optional: pretty timestamp
321
- if "timestamp" in df_display.columns:
322
- df_display["timestamp"] = pd.to_datetime(df_display["timestamp"], unit="s", errors="coerce")
323
 
324
- st.dataframe(
325
- df_display,
326
- column_config={
327
- "Rank": st.column_config.Column("Rank", width="small"),
328
- "team": "Team",
329
- "model": "Model",
330
- "phase_codename": "Phase",
331
- "bbox_mAP": st.column_config.NumberColumn("bbox_mAP", format="%.4f"),
332
- "bbox_AP50": st.column_config.NumberColumn("bbox_AP50", format="%.4f"),
333
- "segm_mAP": st.column_config.NumberColumn("segm_mAP", format="%.4f"),
334
- "segm_AP50": st.column_config.NumberColumn("segm_AP50", format="%.4f"),
335
- "timestamp": st.column_config.DatetimeColumn("Scored at", format="D MMM YYYY, h:mm a"),
336
- },
337
- use_container_width=True,
338
- hide_index=True,
339
- )
340
 
 
341
 
342
- def main():
 
343
 
344
- # st.write("βœ… App booted and rendering UI")
345
-
346
- st.sidebar.title("AI Benchmark Arena πŸ†")
347
 
348
- # Warn early if DB repo isn't configured
349
- if DB_REPO_ID.startswith("NidhiS09/"):
350
- st.sidebar.warning("Set DB_REPO_ID env var or hardcode your private DB dataset repo id in main.py.")
351
 
352
- menu = ["Submit Model", "Leaderboard"]
353
- choice = st.sidebar.radio("Navigation", menu)
354
 
355
- st.sidebar.markdown("---")
356
- st.sidebar.caption("This Space queues submissions to a private DB repo and reads leaderboard results from it.")
 
 
 
 
 
 
 
 
357
 
358
- render_overview()
359
- render_eval_details()
 
360
  st.markdown("---")
 
 
 
 
 
 
361
 
362
- if choice == "Submit Model":
363
- page_submit()
364
- else:
365
- page_leaderboard()
366
 
367
- def render_overview():
368
- with st.expander("ℹ️ Overview of the AI Benchmark Arena"):
369
- st.markdown(
370
- """
371
- **Note:** This Hugging Face Space queues submissions for evaluation and persists results in a private database repo.
372
- """
373
- )
374
- # Keep this optional so missing image doesn't crash the Space
375
- try:
376
- overview_image = Image.open("src/overview_image.png").resize((600, 600))
377
- st.image(overview_image, caption="Example of an object localization task")
378
- except Exception:
379
- st.info("Overview image not found at src/overview_image.png (optional).")
380
 
 
 
 
381
 
382
- def render_eval_details():
383
- with st.expander("πŸ“ How is the Score Calculated?"):
384
- st.markdown(
385
- """
386
- Your submission is evaluated offline by a private evaluator against hidden ground-truth annotations.
387
- The leaderboard reports:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
 
389
- - bbox_mAP
390
- - bbox_AP50
391
- - segm_mAP
392
- - segm_AP50 (default ranking)
393
 
394
- Raw submissions are kept private; only scores and metadata are shown.
395
- """
396
- )
 
 
 
 
 
 
 
397
 
398
 
399
  if __name__ == "__main__":
 
 
 
 
 
1
  import os
2
  import json
3
  import uuid
4
  import time
5
  import tempfile
6
+ import hashlib
7
+ from datetime import datetime, timezone
8
  from typing import Any, Dict, List, Tuple
9
 
10
  import streamlit as st
 
18
  # =========================
19
 
20
  st.set_page_config(
21
+ page_title="VizWiz Benchmark Arena",
22
  page_icon="πŸ†",
23
  layout="wide",
24
  initial_sidebar_state="expanded",
25
  )
26
 
 
27
  DB_REPO_ID = os.getenv("DB_REPO_ID", "NidhiS09/VizWiz-submissions-db")
28
  DB_REPO_TYPE = "dataset"
 
 
 
 
29
  SUBMISSIONS_TOKEN = os.getenv("SUBMISSIONS_TOKEN", "")
30
 
31
+ DAILY_SUBMISSION_CAP = 5
32
+
33
  PHASES = [
34
+ {"label": "Dev (query-dev2024)", "codename": "test-dev2024"},
35
  {"label": "Standard (query-standard2024)", "codename": "test-standard2024"},
36
  {"label": "Challenge (query-challenge2024)", "codename": "test-challenge2024"},
37
  ]
38
 
39
  CHALLENGE_TYPES = ["Object Detection", "Instance Segmentation"]
40
 
 
41
  LEADERBOARD_METRICS = ["bbox_mAP", "bbox_AP50", "segm_mAP", "segm_AP50"]
42
  DEFAULT_SORT_METRIC = "segm_AP50"
43
 
44
  # =========================
45
+ # AUTH HELPERS
46
  # =========================
47
 
48
+ def get_oauth_info() -> Dict | None:
49
+ """
50
+ Returns the HF OAuth user info dict if logged in, else None.
51
+ Keys: preferred_username, name, email, picture, ...
52
+ This works automatically when hf_oauth: true is set in README.md
53
+ """
54
+ try:
55
+ user_info = st.context.headers.get("X-Space-User-Info")
56
+ if user_info:
57
+ return json.loads(user_info)
58
+ except Exception:
59
+ pass
60
+
61
+ # Fallback: check session state (set after OAuth redirect)
62
+ if "oauth_info" in st.session_state and st.session_state["oauth_info"]:
63
+ return st.session_state["oauth_info"]
64
+
65
+ # Use st.experimental_user if available (newer Streamlit on HF Spaces)
66
+ try:
67
+ user = st.experimental_user
68
+ if user and getattr(user, "email", None):
69
+ return {
70
+ "preferred_username": getattr(user, "username", user.email.split("@")[0]),
71
+ "name": getattr(user, "name", user.email),
72
+ "email": user.email,
73
+ "picture": getattr(user, "picture", None),
74
+ }
75
+ except Exception:
76
+ pass
77
+
78
+ return None
79
+
80
+
81
+ def render_login_wall():
82
+ """Shows login prompt and stops rendering if not authenticated."""
83
+ user = get_oauth_info()
84
+ if user:
85
+ return user
86
+
87
+ st.markdown("---")
88
+ st.markdown("## πŸ” Login Required")
89
+ st.info(
90
+ "You need to log in with your Hugging Face account to submit predictions "
91
+ "or view your submission history."
92
+ )
93
+ login_url = os.getenv("SPACE_HOST", "")
94
+ if login_url:
95
+ st.markdown(
96
+ f'<a href="https://{login_url}/oauth/login" target="_self">'
97
+ f'<button style="background:#FF6B35;color:white;border:none;padding:12px 28px;'
98
+ f'font-size:16px;border-radius:8px;cursor:pointer;">πŸ€— Login with Hugging Face</button></a>',
99
+ unsafe_allow_html=True,
100
+ )
101
+ else:
102
+ st.warning(
103
+ "OAuth login URL not available in local dev. "
104
+ "Deploy to HF Spaces with `hf_oauth: true` in README.md."
105
+ )
106
+ st.stop()
107
+
108
+
109
+ # =========================
110
+ # SUBMISSION CAP HELPERS
111
+ # =========================
112
+
113
+ def _today_utc_str() -> str:
114
+ return datetime.now(timezone.utc).strftime("%Y-%m-%d")
115
+
116
+
117
+ def _count_submissions_today(hf_username: str) -> int:
118
+ """
119
+ Count how many submissions this user has made today by scanning submission metadata.
120
+ We do this by listing all submission files and checking meta.json for matching username + today's date.
121
+ Uses a lightweight approach: filter by filename listing only.
122
+ """
123
+ try:
124
+ files = api_client().list_repo_files(
125
+ repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE, token=SUBMISSIONS_TOKEN
126
+ )
127
+ today = _today_utc_str()
128
+ count = 0
129
+ for f in files:
130
+ if not (f.startswith("submissions/") and f.endswith("/meta.json")):
131
+ continue
132
+ try:
133
+ p = hf_hub_download(
134
+ repo_id=DB_REPO_ID,
135
+ repo_type=DB_REPO_TYPE,
136
+ filename=f,
137
+ token=SUBMISSIONS_TOKEN,
138
+ )
139
+ meta = json.load(open(p))
140
+ if meta.get("hf_username") == hf_username:
141
+ ts = meta.get("timestamp", 0)
142
+ sub_date = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
143
+ if sub_date == today:
144
+ count += 1
145
+ except Exception:
146
+ continue
147
+ return count
148
+ except Exception:
149
+ return 0
150
+
151
+
152
+ # =========================
153
+ # HF API HELPERS
154
+ # =========================
155
+
156
+ @st.cache_resource
157
+ def api_client() -> HfApi:
158
+ return HfApi()
159
+
160
+
161
  def _require_token() -> None:
162
  if not SUBMISSIONS_TOKEN:
163
  st.error(
164
+ "Missing SUBMISSIONS_TOKEN. Add it in Space Settings β†’ Secrets."
 
165
  )
166
  st.stop()
167
 
168
 
169
  def _validate_submission_json(obj: Any) -> Tuple[bool, str]:
 
 
 
 
 
 
 
170
  if not isinstance(obj, list):
171
  return False, "Submission must be a JSON list of annotations."
172
 
 
175
  for i, ann in enumerate(obj):
176
  if not isinstance(ann, dict):
177
  return False, f"Annotation at index {i} must be an object/dict."
 
178
  missing = required_keys - set(ann.keys())
179
  if missing:
180
  return False, f"Annotation at index {i} missing keys: {sorted(list(missing))}"
 
 
181
  if not isinstance(ann["image_id"], int):
182
  return False, f"image_id at index {i} must be an integer."
183
  if not isinstance(ann["category_id"], int):
184
  return False, f"category_id at index {i} must be an integer."
 
185
  if not isinstance(ann["score"], (int, float)):
186
  return False, f"score at index {i} must be a number."
 
187
  if not isinstance(ann["area"], (int, float)):
188
  return False, f"area at index {i} must be a number."
 
189
  bbox = ann["bbox"]
190
  if not (isinstance(bbox, list) and len(bbox) == 4 and all(isinstance(x, (int, float)) for x in bbox)):
191
  return False, f"bbox at index {i} must be a list of 4 numbers: [x, y, w, h]."
192
+ if not isinstance(ann["segmentation"], list):
 
 
193
  return False, f"segmentation at index {i} must be a list."
194
 
195
  return True, "OK"
196
 
197
 
198
+ def _upload_json(data: Any, path_in_repo: str, commit_message: str = "") -> None:
199
  with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp:
200
  json.dump(data, tmp, ensure_ascii=False)
201
  tmp_path = tmp.name
 
202
  try:
203
+ api_client().upload_file(
204
  path_or_fileobj=tmp_path,
205
  path_in_repo=path_in_repo,
206
  repo_id=DB_REPO_ID,
207
  repo_type=DB_REPO_TYPE,
208
  token=SUBMISSIONS_TOKEN,
209
+ commit_message=commit_message or f"Add {path_in_repo}",
210
  )
211
  finally:
212
  try:
 
223
  phase_codename: str,
224
  challenge_type: str,
225
  original_filename: str,
226
+ hf_username: str,
227
+ hf_email: str,
228
  ) -> str:
 
 
 
 
229
  _require_token()
 
230
 
231
  submission_id = str(uuid.uuid4())
232
  ts = int(time.time())
 
239
  "challenge_type": challenge_type,
240
  "timestamp": ts,
241
  "original_filename": original_filename,
242
+ "hf_username": hf_username,
243
+ "hf_email": hf_email,
244
  }
245
 
246
  status = {"state": "queued", "timestamp": ts}
 
247
  base = f"submissions/{submission_id}"
248
+
249
+ _upload_json(pred, f"{base}/pred.json", f"pred {submission_id}")
250
+ _upload_json(meta, f"{base}/meta.json", f"meta {submission_id}")
251
+ _upload_json(status, f"{base}/status.json", f"status {submission_id}")
252
 
253
  return submission_id
254
 
255
 
256
+ def _load_leaderboard_df() -> pd.DataFrame:
 
 
 
 
257
  _require_token()
258
  try:
259
+ path = hf_hub_download(
260
  repo_id=DB_REPO_ID,
261
  repo_type=DB_REPO_TYPE,
262
  filename="leaderboard.jsonl",
263
  token=SUBMISSIONS_TOKEN,
264
  )
265
  except HfHubHTTPError as e:
 
266
  if "404" in str(e):
267
+ return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
268
  raise
269
 
 
 
 
 
 
 
270
  rows = []
271
  with open(path, "r", encoding="utf-8") as f:
272
  for line in f:
 
276
  try:
277
  rows.append(json.loads(line))
278
  except json.JSONDecodeError:
 
279
  continue
280
 
281
  if not rows:
282
  return pd.DataFrame(columns=["team", "model", "phase_codename", *LEADERBOARD_METRICS, "timestamp"])
283
 
284
  df = pd.DataFrame(rows)
 
 
285
  for col in ["team", "model", "phase_codename", "timestamp", *LEADERBOARD_METRICS]:
286
  if col not in df.columns:
287
  df[col] = None
288
 
 
289
  if DEFAULT_SORT_METRIC in df.columns:
290
  df = df.sort_values(by=DEFAULT_SORT_METRIC, ascending=False, kind="mergesort")
291
 
292
  return df
293
 
294
 
295
+ def _load_user_submissions(hf_username: str) -> List[Dict]:
296
+ """Load all submissions for a specific HF user."""
297
+ _require_token()
298
+ try:
299
+ files = api_client().list_repo_files(
300
+ repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE, token=SUBMISSIONS_TOKEN
301
+ )
302
+ except Exception:
303
+ return []
304
+
305
+ results = []
306
+ for f in files:
307
+ if not (f.startswith("submissions/") and f.endswith("/meta.json")):
308
+ continue
309
+ try:
310
+ sid = f.split("/")[1]
311
+ meta_path = hf_hub_download(
312
+ repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
313
+ filename=f, token=SUBMISSIONS_TOKEN
314
+ )
315
+ meta = json.load(open(meta_path))
316
+
317
+ if meta.get("hf_username") != hf_username:
318
+ continue
319
+
320
+ # Get status
321
+ try:
322
+ status_path = hf_hub_download(
323
+ repo_id=DB_REPO_ID, repo_type=DB_REPO_TYPE,
324
+ filename=f"submissions/{sid}/status.json", token=SUBMISSIONS_TOKEN
325
+ )
326
+ status = json.load(open(status_path))
327
+ except Exception:
328
+ status = {"state": "unknown"}
329
+
330
+ # Get results if done
331
+ metrics = {}
332
+ if status.get("state") == "done":
333
+ metrics = status.get("metrics", {})
334
+
335
+ results.append({
336
+ "submission_id": sid,
337
+ "team": meta.get("team", ""),
338
+ "model": meta.get("model", ""),
339
+ "phase": meta.get("phase_codename", ""),
340
+ "challenge_type": meta.get("challenge_type", ""),
341
+ "timestamp": meta.get("timestamp", 0),
342
+ "state": status.get("state", "unknown"),
343
+ **metrics,
344
+ })
345
+ except Exception:
346
+ continue
347
+
348
+ results.sort(key=lambda x: x["timestamp"], reverse=True)
349
+ return results
350
+
351
+
352
  # =========================
353
+ # UI PAGES
354
  # =========================
355
+
356
+ def render_overview():
357
+ with st.expander("ℹ️ Overview of the VizWiz Benchmark Arena"):
358
+ st.markdown("""
359
+ **VizWiz Object Localization Challenge** β€” Submit your model predictions and get scored automatically.
360
+
361
+ - πŸ“€ **Submit**: Upload your prediction JSON for Dev, Standard, or Challenge phases
362
+ - πŸ† **Leaderboard**: View public rankings for the Standard phase
363
+ - πŸ“‹ **My Submissions**: Track all your past submissions and scores
364
+ """)
365
+ try:
366
+ overview_image = Image.open("src/overview_image.png").resize((600, 600))
367
+ st.image(overview_image, caption="Example of an object localization task")
368
+ except Exception:
369
+ pass
370
+
371
+
372
+ def render_eval_details():
373
+ with st.expander("πŸ“ How is the Score Calculated?"):
374
+ st.markdown("""
375
+ Your submission is evaluated automatically against hidden ground-truth annotations.
376
+ The leaderboard reports:
377
+ - **bbox_mAP** β€” bounding box mean average precision
378
+ - **bbox_AP50** β€” bounding box AP at IoU=0.50
379
+ - **segm_mAP** β€” segmentation mean average precision
380
+ - **segm_AP50** β€” segmentation AP at IoU=0.50 *(default ranking)*
381
+ """)
382
+
383
+
384
+ def page_leaderboard():
385
+ st.header("πŸ† Leaderboard β€” Standard Phase")
386
+ st.caption(f"Ranked by **{DEFAULT_SORT_METRIC}** (descending). Showing Standard phase submissions only.")
387
+
388
+ with st.spinner("Loading leaderboard..."):
389
+ try:
390
+ df = _load_leaderboard_df()
391
+ except Exception as e:
392
+ st.error(f"Could not load leaderboard: {e}")
393
+ return
394
+
395
+ # Filter to standard phase only for public leaderboard
396
+ if not df.empty and "phase_codename" in df.columns:
397
+ df = df[df["phase_codename"] == "test-standard2024"]
398
+
399
+ if df.empty:
400
+ st.info("No scored Standard phase submissions yet. Be the first!")
401
+ return
402
+
403
+ df_display = df.copy()
404
+ df_display.insert(0, "Rank", range(1, len(df_display) + 1))
405
+ if "timestamp" in df_display.columns:
406
+ df_display["timestamp"] = pd.to_datetime(df_display["timestamp"], unit="s", errors="coerce")
407
+
408
+ # Drop hf_username / hf_email from public view
409
+ for col in ["hf_username", "hf_email", "phase_codename", "submission_id"]:
410
+ if col in df_display.columns:
411
+ df_display.drop(columns=[col], inplace=True)
412
+
413
+ st.dataframe(
414
+ df_display,
415
+ column_config={
416
+ "Rank": st.column_config.Column("Rank", width="small"),
417
+ "team": "Team",
418
+ "model": "Model",
419
+ "bbox_mAP": st.column_config.NumberColumn("bbox_mAP", format="%.4f"),
420
+ "bbox_AP50": st.column_config.NumberColumn("bbox_AP50", format="%.4f"),
421
+ "segm_mAP": st.column_config.NumberColumn("segm_mAP", format="%.4f"),
422
+ "segm_AP50": st.column_config.NumberColumn("segm_AP50", format="%.4f"),
423
+ "timestamp": st.column_config.DatetimeColumn("Scored at", format="D MMM YYYY, h:mm a"),
424
+ },
425
+ use_container_width=True,
426
+ hide_index=True,
427
+ )
428
+
429
 
430
  def page_submit():
431
+ user = render_login_wall()
432
+ hf_username = user.get("preferred_username", "unknown")
433
+ hf_email = user.get("email", "")
434
+
435
+ st.header("πŸš€ Submit Your Predictions")
436
+
437
+ # Show user info + daily cap status
438
+ col_info, col_cap = st.columns([3, 1])
439
+ with col_info:
440
+ avatar = user.get("picture", "")
441
+ name = user.get("name", hf_username)
442
+ if avatar:
443
+ st.markdown(
444
+ f'<img src="{avatar}" width="32" style="border-radius:50%;vertical-align:middle;margin-right:8px">'
445
+ f'<b>{name}</b> (@{hf_username})',
446
+ unsafe_allow_html=True,
447
+ )
448
+ else:
449
+ st.markdown(f"πŸ‘€ Logged in as **{name}** (@{hf_username})")
450
+
451
+ with col_cap:
452
+ with st.spinner("Checking daily limit..."):
453
+ subs_today = _count_submissions_today(hf_username)
454
+ remaining = DAILY_SUBMISSION_CAP - subs_today
455
+ color = "green" if remaining > 1 else ("orange" if remaining == 1 else "red")
456
+ st.markdown(
457
+ f'<div style="text-align:center;padding:8px;border-radius:8px;border:1px solid {color}">'
458
+ f'<b style="color:{color}">{remaining}/{DAILY_SUBMISSION_CAP}</b><br>'
459
+ f'<small>submissions left today</small></div>',
460
+ unsafe_allow_html=True,
461
+ )
462
+
463
+ st.markdown("---")
464
+
465
+ if remaining <= 0:
466
+ st.error(f"β›” You've reached your daily limit of {DAILY_SUBMISSION_CAP} submissions. Come back tomorrow!")
467
+ return
468
 
469
  col1, col2 = st.columns([2, 1])
470
 
 
472
  st.subheader("Submission Info")
473
  team = st.text_input("Team / Display Name", value=st.session_state.get("team", ""))
474
  model_name = st.text_input("Model Name", value=st.session_state.get("model_name", ""))
 
475
  phase_label = st.selectbox("Phase", [p["label"] for p in PHASES])
476
  phase_codename = next(p["codename"] for p in PHASES if p["label"] == phase_label)
 
477
  challenge_type = st.radio("Challenge type", CHALLENGE_TYPES, horizontal=False)
 
478
  st.session_state["team"] = team
479
  st.session_state["model_name"] = model_name
480
+ st.caption("Submissions are scored automatically. Results appear in My Submissions once processed.")
 
481
 
482
  with col1:
483
  st.subheader("Upload Submission File")
484
  uploaded_file = st.file_uploader("Choose a JSON file", type=["json"])
485
 
486
  if uploaded_file is None:
487
+ st.info("Upload a JSON file containing a list of annotations.")
488
  return
489
 
 
490
  try:
491
  raw = uploaded_file.getvalue().decode("utf-8")
492
  pred_obj = json.loads(raw)
 
505
 
506
  if submit_clicked:
507
  if not team.strip():
508
+ st.error("Please enter a Team / Display Name.")
509
  return
510
  if not model_name.strip():
511
+ st.error("Please enter a Model Name.")
512
+ return
513
+
514
+ # Re-check cap right before submitting
515
+ subs_today_fresh = _count_submissions_today(hf_username)
516
+ if subs_today_fresh >= DAILY_SUBMISSION_CAP:
517
+ st.error("β›” Daily submission limit reached. Try again tomorrow.")
518
  return
519
 
520
+ with st.spinner("Uploading submission..."):
521
  try:
522
  submission_id = _create_submission_record(
523
  pred=pred_obj,
 
526
  phase_codename=phase_codename,
527
  challenge_type=challenge_type,
528
  original_filename=uploaded_file.name,
529
+ hf_username=hf_username,
530
+ hf_email=hf_email,
531
  )
532
  except Exception as e:
533
  st.error(f"Upload failed: {e}")
 
536
  st.balloons()
537
  st.success("Submission queued successfully!")
538
  st.code(f"Submission ID: {submission_id}")
539
+ st.info("Track your results in **My Submissions** once the evaluator processes your file.")
540
 
 
541
 
542
+ def page_my_submissions():
543
+ user = render_login_wall()
544
+ hf_username = user.get("preferred_username", "unknown")
545
 
546
+ st.header("πŸ“‹ My Submissions")
547
+ st.caption(f"Showing all submissions for @{hf_username}")
 
548
 
549
+ with st.spinner("Loading your submissions..."):
550
+ submissions = _load_user_submissions(hf_username)
 
 
 
 
551
 
552
+ if not submissions:
553
+ st.info("You haven't made any submissions yet. Head to **Submit Model** to get started!")
554
  return
555
 
556
+ # Phase filter
557
+ phases_present = sorted(set(s["phase"] for s in submissions))
558
+ selected_phase = st.selectbox("Filter by phase", ["All"] + phases_present)
559
 
560
+ filtered = submissions if selected_phase == "All" else [s for s in submissions if s["phase"] == selected_phase]
 
 
561
 
562
+ # State badges
563
+ state_colors = {
564
+ "queued": "🟑",
565
+ "running": "πŸ”΅",
566
+ "done": "🟒",
567
+ "failed": "πŸ”΄",
568
+ "unknown": "βšͺ",
569
+ }
 
 
 
 
 
 
 
 
570
 
571
+ df = pd.DataFrame(filtered)
572
 
573
+ if "timestamp" in df.columns:
574
+ df["submitted_at"] = pd.to_datetime(df["timestamp"], unit="s", errors="coerce")
575
 
576
+ if "state" in df.columns:
577
+ df["status"] = df["state"].apply(lambda s: f"{state_colors.get(s, 'βšͺ')} {s.capitalize()}")
 
578
 
579
+ display_cols = ["submitted_at", "status", "team", "model", "phase", "challenge_type"]
580
+ metric_cols = [m for m in LEADERBOARD_METRICS if m in df.columns]
581
+ display_cols += metric_cols
582
 
583
+ df_display = df[[c for c in display_cols if c in df.columns]].copy()
 
584
 
585
+ col_config = {
586
+ "submitted_at": st.column_config.DatetimeColumn("Submitted At", format="D MMM YYYY, h:mm a"),
587
+ "status": "Status",
588
+ "team": "Team",
589
+ "model": "Model",
590
+ "phase": "Phase",
591
+ "challenge_type": "Challenge Type",
592
+ }
593
+ for m in metric_cols:
594
+ col_config[m] = st.column_config.NumberColumn(m, format="%.4f")
595
 
596
+ st.dataframe(df_display, column_config=col_config, use_container_width=True, hide_index=True)
597
+
598
+ # Summary stats
599
  st.markdown("---")
600
+ total = len(submissions)
601
+ done = sum(1 for s in submissions if s["state"] == "done")
602
+ today_count = sum(
603
+ 1 for s in submissions
604
+ if datetime.fromtimestamp(s["timestamp"], tz=timezone.utc).strftime("%Y-%m-%d") == _today_utc_str()
605
+ )
606
 
607
+ c1, c2, c3 = st.columns(3)
608
+ c1.metric("Total Submissions", total)
609
+ c2.metric("Scored", done)
610
+ c3.metric(f"Today ({DAILY_SUBMISSION_CAP} max)", f"{today_count}/{DAILY_SUBMISSION_CAP}")
611
 
 
 
 
 
 
 
 
 
 
 
 
 
 
612
 
613
+ # =========================
614
+ # MAIN
615
+ # =========================
616
 
617
+ def main():
618
+ st.sidebar.title("VizWiz Benchmark πŸ†")
619
+
620
+ # Check login state for sidebar display
621
+ user = get_oauth_info()
622
+
623
+ if user:
624
+ hf_username = user.get("preferred_username", "")
625
+ avatar = user.get("picture", "")
626
+ if avatar:
627
+ st.sidebar.markdown(
628
+ f'<img src="{avatar}" width="40" style="border-radius:50%"><br>'
629
+ f'<b>@{hf_username}</b>',
630
+ unsafe_allow_html=True,
631
+ )
632
+ else:
633
+ st.sidebar.markdown(f"πŸ‘€ @{hf_username}")
634
+ st.sidebar.markdown("---")
635
+ menu = ["Leaderboard", "Submit Model", "My Submissions"]
636
+ else:
637
+ st.sidebar.info("Login to submit predictions and view your history.")
638
+ login_url = os.getenv("SPACE_HOST", "")
639
+ if login_url:
640
+ st.sidebar.markdown(
641
+ f'<a href="https://{login_url}/oauth/login" target="_self">'
642
+ f'<button style="width:100%;background:#FF6B35;color:white;border:none;'
643
+ f'padding:10px;border-radius:6px;cursor:pointer;font-size:14px;">πŸ€— Login with HF</button></a>',
644
+ unsafe_allow_html=True,
645
+ )
646
+ menu = ["Leaderboard", "Submit Model", "My Submissions"]
647
 
648
+ choice = st.sidebar.radio("Navigation", menu)
649
+ st.sidebar.markdown("---")
650
+ st.sidebar.caption("Submissions are queued to a private DB repo and evaluated automatically.")
 
651
 
652
+ render_overview()
653
+ render_eval_details()
654
+ st.markdown("---")
655
+
656
+ if choice == "Leaderboard":
657
+ page_leaderboard()
658
+ elif choice == "Submit Model":
659
+ page_submit()
660
+ elif choice == "My Submissions":
661
+ page_my_submissions()
662
 
663
 
664
  if __name__ == "__main__":