fffiloni commited on
Commit
413f89f
·
verified ·
1 Parent(s): ca46a92

fix: prevent infinite fetch by stopping after crossing year boundary

Browse files
Files changed (1) hide show
  1. app.py +61 -46
app.py CHANGED
@@ -1,9 +1,9 @@
1
  import html
2
- import threading
3
  import time
4
- from concurrent.futures import ThreadPoolExecutor, as_completed
5
  from datetime import datetime
6
  from functools import lru_cache
 
7
 
8
  import gradio as gr
9
  import pandas as pd
@@ -33,8 +33,6 @@ REPO_KIND_PALETTES = {
33
 
34
  RECENT_ACTIVITY_LIMIT = 20
35
  RECENT_ACTIVITY_MAX_PAGES = 2000
36
- RECENT_ACTIVITY_REQUEST_TIMEOUT = 20
37
- RECENT_ACTIVITY_OPEN_EVENT_EPSILON_SECONDS = 90
38
 
39
 
40
  # ============================================================
@@ -68,7 +66,7 @@ def normalize_recent_activity_payload(payload):
68
  if isinstance(events, list):
69
  return events, payload.get("cursor")
70
 
71
- for key in ("items", "events", "activity"):
72
  value = payload.get(key)
73
  if isinstance(value, list):
74
  return value, payload.get("cursor")
@@ -82,20 +80,10 @@ def get_year_bounds(selected_year: int):
82
  return start_ts, end_ts
83
 
84
 
85
- def should_stop_after_year_coverage(
86
- seen_target_year: bool,
87
- passed_before_target_year: bool,
88
  ) -> bool:
89
- return seen_target_year and passed_before_target_year
90
-
91
-
92
- def parse_utc_ts(raw_value):
93
- if not raw_value:
94
- return None
95
- try:
96
- return pd.to_datetime(raw_value, utc=True)
97
- except Exception:
98
- return None
99
 
100
 
101
  # ============================================================
@@ -527,11 +515,13 @@ def fetch_user_discussion_activity(
527
  pages_fetched = 0
528
 
529
  seen_target_year = False
530
- passed_before_target_year = False
531
  oldest_event_ts_seen = None
532
 
533
- year_start, year_end = get_year_bounds(selected_year)
534
  logs = []
 
 
535
  session = build_session("application/json")
536
 
537
  def extract_author_name(discussion: dict) -> str | None:
@@ -539,12 +529,20 @@ def fetch_user_discussion_activity(
539
  if isinstance(author, str):
540
  return author
541
  if isinstance(author, dict):
542
- for key in ("name", "username", "login"):
543
  value = author.get(key)
544
  if isinstance(value, str):
545
  return value
546
  return None
547
 
 
 
 
 
 
 
 
 
548
  def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
549
  is_pr = bool(discussion.get("isPullRequest", False))
550
  author_name = extract_author_name(discussion)
@@ -579,8 +577,8 @@ def fetch_user_discussion_activity(
579
  return False, None
580
 
581
  def extract_timestamp_for_mode(event: dict, discussion: dict):
582
- event_ts = parse_utc_ts(event.get("time"))
583
- created_ts = parse_utc_ts(discussion.get("createdAt"))
584
 
585
  if mode in {"pr_opened", "thread_opened"}:
586
  return created_ts
@@ -590,7 +588,7 @@ def fetch_user_discussion_activity(
590
  return None
591
  if created_ts is not None:
592
  delta_seconds = abs((event_ts - created_ts).total_seconds())
593
- if delta_seconds <= RECENT_ACTIVITY_OPEN_EVENT_EPSILON_SECONDS:
594
  return None
595
  return event_ts
596
 
@@ -618,7 +616,7 @@ def fetch_user_discussion_activity(
618
  response = session.get(
619
  "https://huggingface.co/api/recent-activity",
620
  params=params,
621
- timeout=RECENT_ACTIVITY_REQUEST_TIMEOUT,
622
  )
623
  response.raise_for_status()
624
 
@@ -635,7 +633,7 @@ def fetch_user_discussion_activity(
635
  if not isinstance(event, dict):
636
  continue
637
 
638
- event_ts = parse_utc_ts(event.get("time"))
639
  if event_ts is not None:
640
  if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
641
  oldest_event_ts_seen = event_ts
@@ -643,7 +641,7 @@ def fetch_user_discussion_activity(
643
  if year_start <= event_ts <= year_end:
644
  seen_target_year = True
645
  elif event_ts < year_start:
646
- passed_before_target_year = True
647
 
648
  discussion = event.get("discussionData") or {}
649
  if not isinstance(discussion, dict):
@@ -670,8 +668,8 @@ def fetch_user_discussion_activity(
670
  seen_ids.add(unique_id)
671
  dates.append(dt.tz_localize(None).date())
672
 
673
- if should_stop_after_year_coverage(seen_target_year, passed_before_target_year):
674
- logs.append("Stopped after fully covering selected year.")
675
  break
676
 
677
  if next_cursor:
@@ -683,7 +681,7 @@ def fetch_user_discussion_activity(
683
  if oldest_event_ts_seen is not None:
684
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
685
  logs.append(f"Seen target year: {seen_target_year}.")
686
- logs.append(f"Passed before target year: {passed_before_target_year}.")
687
  logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
688
 
689
  return {
@@ -786,11 +784,12 @@ def fetch_bucketed_recent_activity(
786
  pages_fetched = 0
787
 
788
  seen_target_year = False
789
- passed_before_target_year = False
790
  oldest_event_ts_seen = None
791
 
792
- year_start, year_end = get_year_bounds(selected_year)
793
  logs = []
 
 
794
  session = build_session("application/json")
795
 
796
  try:
@@ -815,7 +814,7 @@ def fetch_bucketed_recent_activity(
815
  response = session.get(
816
  "https://huggingface.co/api/recent-activity",
817
  params=params,
818
- timeout=RECENT_ACTIVITY_REQUEST_TIMEOUT,
819
  )
820
  response.raise_for_status()
821
 
@@ -832,8 +831,13 @@ def fetch_bucketed_recent_activity(
832
  if not isinstance(event, dict):
833
  continue
834
 
835
- event_ts = parse_utc_ts(event.get("time"))
836
- if event_ts is None:
 
 
 
 
 
837
  continue
838
 
839
  if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
@@ -842,26 +846,30 @@ def fetch_bucketed_recent_activity(
842
  if year_start <= event_ts <= year_end:
843
  seen_target_year = True
844
  elif event_ts < year_start:
845
- passed_before_target_year = True
846
 
847
  if event_ts > year_end:
848
  continue
 
849
  if event_ts < year_start:
850
  continue
851
 
852
  bucket_key, unique_id = classifier(event)
 
853
  if not bucket_key or not unique_id:
854
  continue
 
855
  if bucket_key not in bucket_dates:
856
  continue
 
857
  if unique_id in bucket_seen_ids[bucket_key]:
858
  continue
859
 
860
  bucket_seen_ids[bucket_key].add(unique_id)
861
  bucket_dates[bucket_key].append(event_ts.tz_localize(None).date())
862
 
863
- if should_stop_after_year_coverage(seen_target_year, passed_before_target_year):
864
- logs.append("Stopped after fully covering selected year.")
865
  break
866
 
867
  if next_cursor:
@@ -888,7 +896,7 @@ def fetch_bucketed_recent_activity(
888
  if oldest_event_ts_seen is not None:
889
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
890
  logs.append(f"Seen target year: {seen_target_year}.")
891
- logs.append(f"Passed before target year: {passed_before_target_year}.")
892
  logs.append(
893
  " | ".join(
894
  f"{bucket_titles[key]}: {len(bucket_dates[key])}"
@@ -897,6 +905,7 @@ def fetch_bucketed_recent_activity(
897
  )
898
 
899
  result["logs"] = "\n".join(logs)
 
900
  set_cache(cache_key, result)
901
  return result
902
 
@@ -1735,7 +1744,12 @@ def render_invalid_app_state(app_state: dict | None):
1735
  error_text = (app_state or {}).get("validation_error") or f"Username or org '{username}' was not found."
1736
 
1737
  empty_counts = {kind: 0 for kind in REPO_KINDS}
1738
- empty_metrics = render_metric_cards(0, empty_counts, empty_counts, selected_year)
 
 
 
 
 
1739
 
1740
  return (
1741
  "## Ready",
@@ -1757,7 +1771,12 @@ def render_app_state(
1757
  if not app_state or not app_state.get("username"):
1758
  selected_year = datetime.now().year
1759
  empty_counts = {kind: 0 for kind in REPO_KINDS}
1760
- empty_metrics = render_metric_cards(0, empty_counts, empty_counts, selected_year)
 
 
 
 
 
1761
  context_html = render_idle_context_banner("", profile)
1762
  return (
1763
  "## Ready",
@@ -1915,11 +1934,7 @@ def fetch_and_render_repo_only(
1915
  logs_box,
1916
  )
1917
 
1918
- requested_visibility = (
1919
- current_visibility_mode
1920
- if current_visibility_mode in {"all", "public"}
1921
- else app_state["default_visibility"]
1922
- )
1923
  final_visibility = requested_visibility if app_state["can_toggle_visibility"] else "public"
1924
 
1925
  title_md, context_box, metrics_box, all_chart, model_chart, dataset_chart, space_chart, logs_box = render_app_state(
 
1
  import html
 
2
  import time
3
+ import threading
4
  from datetime import datetime
5
  from functools import lru_cache
6
+ from concurrent.futures import ThreadPoolExecutor, as_completed
7
 
8
  import gradio as gr
9
  import pandas as pd
 
33
 
34
  RECENT_ACTIVITY_LIMIT = 20
35
  RECENT_ACTIVITY_MAX_PAGES = 2000
 
 
36
 
37
 
38
  # ============================================================
 
66
  if isinstance(events, list):
67
  return events, payload.get("cursor")
68
 
69
+ for key in ["items", "events", "activity"]:
70
  value = payload.get(key)
71
  if isinstance(value, list):
72
  return value, payload.get("cursor")
 
80
  return start_ts, end_ts
81
 
82
 
83
+ def should_stop_after_crossing_year_start(
84
+ crossed_before_year_start: bool,
 
85
  ) -> bool:
86
+ return crossed_before_year_start
 
 
 
 
 
 
 
 
 
87
 
88
 
89
  # ============================================================
 
515
  pages_fetched = 0
516
 
517
  seen_target_year = False
518
+ crossed_before_year_start = False
519
  oldest_event_ts_seen = None
520
 
521
+ OPEN_EVENT_EPSILON_SECONDS = 90
522
  logs = []
523
+
524
+ year_start, year_end = get_year_bounds(selected_year)
525
  session = build_session("application/json")
526
 
527
  def extract_author_name(discussion: dict) -> str | None:
 
529
  if isinstance(author, str):
530
  return author
531
  if isinstance(author, dict):
532
+ for key in ["name", "username", "login"]:
533
  value = author.get(key)
534
  if isinstance(value, str):
535
  return value
536
  return None
537
 
538
+ def parse_ts(raw_value):
539
+ if not raw_value:
540
+ return None
541
+ try:
542
+ return pd.to_datetime(raw_value, utc=True)
543
+ except Exception:
544
+ return None
545
+
546
  def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
547
  is_pr = bool(discussion.get("isPullRequest", False))
548
  author_name = extract_author_name(discussion)
 
577
  return False, None
578
 
579
  def extract_timestamp_for_mode(event: dict, discussion: dict):
580
+ event_ts = parse_ts(event.get("time"))
581
+ created_ts = parse_ts(discussion.get("createdAt"))
582
 
583
  if mode in {"pr_opened", "thread_opened"}:
584
  return created_ts
 
588
  return None
589
  if created_ts is not None:
590
  delta_seconds = abs((event_ts - created_ts).total_seconds())
591
+ if delta_seconds <= OPEN_EVENT_EPSILON_SECONDS:
592
  return None
593
  return event_ts
594
 
 
616
  response = session.get(
617
  "https://huggingface.co/api/recent-activity",
618
  params=params,
619
+ timeout=20,
620
  )
621
  response.raise_for_status()
622
 
 
633
  if not isinstance(event, dict):
634
  continue
635
 
636
+ event_ts = parse_ts(event.get("time"))
637
  if event_ts is not None:
638
  if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
639
  oldest_event_ts_seen = event_ts
 
641
  if year_start <= event_ts <= year_end:
642
  seen_target_year = True
643
  elif event_ts < year_start:
644
+ crossed_before_year_start = True
645
 
646
  discussion = event.get("discussionData") or {}
647
  if not isinstance(discussion, dict):
 
668
  seen_ids.add(unique_id)
669
  dates.append(dt.tz_localize(None).date())
670
 
671
+ if should_stop_after_crossing_year_start(crossed_before_year_start):
672
+ logs.append("Stopped after crossing before selected year.")
673
  break
674
 
675
  if next_cursor:
 
681
  if oldest_event_ts_seen is not None:
682
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
683
  logs.append(f"Seen target year: {seen_target_year}.")
684
+ logs.append(f"Crossed before year start: {crossed_before_year_start}.")
685
  logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
686
 
687
  return {
 
784
  pages_fetched = 0
785
 
786
  seen_target_year = False
787
+ crossed_before_year_start = False
788
  oldest_event_ts_seen = None
789
 
 
790
  logs = []
791
+
792
+ year_start, year_end = get_year_bounds(selected_year)
793
  session = build_session("application/json")
794
 
795
  try:
 
814
  response = session.get(
815
  "https://huggingface.co/api/recent-activity",
816
  params=params,
817
+ timeout=20,
818
  )
819
  response.raise_for_status()
820
 
 
831
  if not isinstance(event, dict):
832
  continue
833
 
834
+ raw_time = event.get("time")
835
+ if not raw_time:
836
+ continue
837
+
838
+ try:
839
+ event_ts = pd.to_datetime(raw_time, utc=True)
840
+ except Exception:
841
  continue
842
 
843
  if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
 
846
  if year_start <= event_ts <= year_end:
847
  seen_target_year = True
848
  elif event_ts < year_start:
849
+ crossed_before_year_start = True
850
 
851
  if event_ts > year_end:
852
  continue
853
+
854
  if event_ts < year_start:
855
  continue
856
 
857
  bucket_key, unique_id = classifier(event)
858
+
859
  if not bucket_key or not unique_id:
860
  continue
861
+
862
  if bucket_key not in bucket_dates:
863
  continue
864
+
865
  if unique_id in bucket_seen_ids[bucket_key]:
866
  continue
867
 
868
  bucket_seen_ids[bucket_key].add(unique_id)
869
  bucket_dates[bucket_key].append(event_ts.tz_localize(None).date())
870
 
871
+ if should_stop_after_crossing_year_start(crossed_before_year_start):
872
+ logs.append("Stopped after crossing before selected year.")
873
  break
874
 
875
  if next_cursor:
 
896
  if oldest_event_ts_seen is not None:
897
  logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
898
  logs.append(f"Seen target year: {seen_target_year}.")
899
+ logs.append(f"Crossed before year start: {crossed_before_year_start}.")
900
  logs.append(
901
  " | ".join(
902
  f"{bucket_titles[key]}: {len(bucket_dates[key])}"
 
905
  )
906
 
907
  result["logs"] = "\n".join(logs)
908
+
909
  set_cache(cache_key, result)
910
  return result
911
 
 
1744
  error_text = (app_state or {}).get("validation_error") or f"Username or org '{username}' was not found."
1745
 
1746
  empty_counts = {kind: 0 for kind in REPO_KINDS}
1747
+ empty_metrics = render_metric_cards(
1748
+ 0,
1749
+ empty_counts,
1750
+ empty_counts,
1751
+ selected_year,
1752
+ )
1753
 
1754
  return (
1755
  "## Ready",
 
1771
  if not app_state or not app_state.get("username"):
1772
  selected_year = datetime.now().year
1773
  empty_counts = {kind: 0 for kind in REPO_KINDS}
1774
+ empty_metrics = render_metric_cards(
1775
+ 0,
1776
+ empty_counts,
1777
+ empty_counts,
1778
+ selected_year,
1779
+ )
1780
  context_html = render_idle_context_banner("", profile)
1781
  return (
1782
  "## Ready",
 
1934
  logs_box,
1935
  )
1936
 
1937
+ requested_visibility = current_visibility_mode if current_visibility_mode in {"all", "public"} else app_state["default_visibility"]
 
 
 
 
1938
  final_visibility = requested_visibility if app_state["can_toggle_visibility"] else "public"
1939
 
1940
  title_md, context_box, metrics_box, all_chart, model_chart, dataset_chart, space_chart, logs_box = render_app_state(