Spaces:
Sleeping
Sleeping
fix: prevent infinite fetch by stopping after crossing year boundary
Browse files
app.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import html
|
| 2 |
-
import threading
|
| 3 |
import time
|
| 4 |
-
|
| 5 |
from datetime import datetime
|
| 6 |
from functools import lru_cache
|
|
|
|
| 7 |
|
| 8 |
import gradio as gr
|
| 9 |
import pandas as pd
|
|
@@ -33,8 +33,6 @@ REPO_KIND_PALETTES = {
|
|
| 33 |
|
| 34 |
RECENT_ACTIVITY_LIMIT = 20
|
| 35 |
RECENT_ACTIVITY_MAX_PAGES = 2000
|
| 36 |
-
RECENT_ACTIVITY_REQUEST_TIMEOUT = 20
|
| 37 |
-
RECENT_ACTIVITY_OPEN_EVENT_EPSILON_SECONDS = 90
|
| 38 |
|
| 39 |
|
| 40 |
# ============================================================
|
|
@@ -68,7 +66,7 @@ def normalize_recent_activity_payload(payload):
|
|
| 68 |
if isinstance(events, list):
|
| 69 |
return events, payload.get("cursor")
|
| 70 |
|
| 71 |
-
for key in
|
| 72 |
value = payload.get(key)
|
| 73 |
if isinstance(value, list):
|
| 74 |
return value, payload.get("cursor")
|
|
@@ -82,20 +80,10 @@ def get_year_bounds(selected_year: int):
|
|
| 82 |
return start_ts, end_ts
|
| 83 |
|
| 84 |
|
| 85 |
-
def
|
| 86 |
-
|
| 87 |
-
passed_before_target_year: bool,
|
| 88 |
) -> bool:
|
| 89 |
-
return
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
def parse_utc_ts(raw_value):
|
| 93 |
-
if not raw_value:
|
| 94 |
-
return None
|
| 95 |
-
try:
|
| 96 |
-
return pd.to_datetime(raw_value, utc=True)
|
| 97 |
-
except Exception:
|
| 98 |
-
return None
|
| 99 |
|
| 100 |
|
| 101 |
# ============================================================
|
|
@@ -527,11 +515,13 @@ def fetch_user_discussion_activity(
|
|
| 527 |
pages_fetched = 0
|
| 528 |
|
| 529 |
seen_target_year = False
|
| 530 |
-
|
| 531 |
oldest_event_ts_seen = None
|
| 532 |
|
| 533 |
-
|
| 534 |
logs = []
|
|
|
|
|
|
|
| 535 |
session = build_session("application/json")
|
| 536 |
|
| 537 |
def extract_author_name(discussion: dict) -> str | None:
|
|
@@ -539,12 +529,20 @@ def fetch_user_discussion_activity(
|
|
| 539 |
if isinstance(author, str):
|
| 540 |
return author
|
| 541 |
if isinstance(author, dict):
|
| 542 |
-
for key in
|
| 543 |
value = author.get(key)
|
| 544 |
if isinstance(value, str):
|
| 545 |
return value
|
| 546 |
return None
|
| 547 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 548 |
def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
|
| 549 |
is_pr = bool(discussion.get("isPullRequest", False))
|
| 550 |
author_name = extract_author_name(discussion)
|
|
@@ -579,8 +577,8 @@ def fetch_user_discussion_activity(
|
|
| 579 |
return False, None
|
| 580 |
|
| 581 |
def extract_timestamp_for_mode(event: dict, discussion: dict):
|
| 582 |
-
event_ts =
|
| 583 |
-
created_ts =
|
| 584 |
|
| 585 |
if mode in {"pr_opened", "thread_opened"}:
|
| 586 |
return created_ts
|
|
@@ -590,7 +588,7 @@ def fetch_user_discussion_activity(
|
|
| 590 |
return None
|
| 591 |
if created_ts is not None:
|
| 592 |
delta_seconds = abs((event_ts - created_ts).total_seconds())
|
| 593 |
-
if delta_seconds <=
|
| 594 |
return None
|
| 595 |
return event_ts
|
| 596 |
|
|
@@ -618,7 +616,7 @@ def fetch_user_discussion_activity(
|
|
| 618 |
response = session.get(
|
| 619 |
"https://huggingface.co/api/recent-activity",
|
| 620 |
params=params,
|
| 621 |
-
timeout=
|
| 622 |
)
|
| 623 |
response.raise_for_status()
|
| 624 |
|
|
@@ -635,7 +633,7 @@ def fetch_user_discussion_activity(
|
|
| 635 |
if not isinstance(event, dict):
|
| 636 |
continue
|
| 637 |
|
| 638 |
-
event_ts =
|
| 639 |
if event_ts is not None:
|
| 640 |
if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
|
| 641 |
oldest_event_ts_seen = event_ts
|
|
@@ -643,7 +641,7 @@ def fetch_user_discussion_activity(
|
|
| 643 |
if year_start <= event_ts <= year_end:
|
| 644 |
seen_target_year = True
|
| 645 |
elif event_ts < year_start:
|
| 646 |
-
|
| 647 |
|
| 648 |
discussion = event.get("discussionData") or {}
|
| 649 |
if not isinstance(discussion, dict):
|
|
@@ -670,8 +668,8 @@ def fetch_user_discussion_activity(
|
|
| 670 |
seen_ids.add(unique_id)
|
| 671 |
dates.append(dt.tz_localize(None).date())
|
| 672 |
|
| 673 |
-
if
|
| 674 |
-
logs.append("Stopped after
|
| 675 |
break
|
| 676 |
|
| 677 |
if next_cursor:
|
|
@@ -683,7 +681,7 @@ def fetch_user_discussion_activity(
|
|
| 683 |
if oldest_event_ts_seen is not None:
|
| 684 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
| 685 |
logs.append(f"Seen target year: {seen_target_year}.")
|
| 686 |
-
logs.append(f"
|
| 687 |
logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
|
| 688 |
|
| 689 |
return {
|
|
@@ -786,11 +784,12 @@ def fetch_bucketed_recent_activity(
|
|
| 786 |
pages_fetched = 0
|
| 787 |
|
| 788 |
seen_target_year = False
|
| 789 |
-
|
| 790 |
oldest_event_ts_seen = None
|
| 791 |
|
| 792 |
-
year_start, year_end = get_year_bounds(selected_year)
|
| 793 |
logs = []
|
|
|
|
|
|
|
| 794 |
session = build_session("application/json")
|
| 795 |
|
| 796 |
try:
|
|
@@ -815,7 +814,7 @@ def fetch_bucketed_recent_activity(
|
|
| 815 |
response = session.get(
|
| 816 |
"https://huggingface.co/api/recent-activity",
|
| 817 |
params=params,
|
| 818 |
-
timeout=
|
| 819 |
)
|
| 820 |
response.raise_for_status()
|
| 821 |
|
|
@@ -832,8 +831,13 @@ def fetch_bucketed_recent_activity(
|
|
| 832 |
if not isinstance(event, dict):
|
| 833 |
continue
|
| 834 |
|
| 835 |
-
|
| 836 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 837 |
continue
|
| 838 |
|
| 839 |
if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
|
|
@@ -842,26 +846,30 @@ def fetch_bucketed_recent_activity(
|
|
| 842 |
if year_start <= event_ts <= year_end:
|
| 843 |
seen_target_year = True
|
| 844 |
elif event_ts < year_start:
|
| 845 |
-
|
| 846 |
|
| 847 |
if event_ts > year_end:
|
| 848 |
continue
|
|
|
|
| 849 |
if event_ts < year_start:
|
| 850 |
continue
|
| 851 |
|
| 852 |
bucket_key, unique_id = classifier(event)
|
|
|
|
| 853 |
if not bucket_key or not unique_id:
|
| 854 |
continue
|
|
|
|
| 855 |
if bucket_key not in bucket_dates:
|
| 856 |
continue
|
|
|
|
| 857 |
if unique_id in bucket_seen_ids[bucket_key]:
|
| 858 |
continue
|
| 859 |
|
| 860 |
bucket_seen_ids[bucket_key].add(unique_id)
|
| 861 |
bucket_dates[bucket_key].append(event_ts.tz_localize(None).date())
|
| 862 |
|
| 863 |
-
if
|
| 864 |
-
logs.append("Stopped after
|
| 865 |
break
|
| 866 |
|
| 867 |
if next_cursor:
|
|
@@ -888,7 +896,7 @@ def fetch_bucketed_recent_activity(
|
|
| 888 |
if oldest_event_ts_seen is not None:
|
| 889 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
| 890 |
logs.append(f"Seen target year: {seen_target_year}.")
|
| 891 |
-
logs.append(f"
|
| 892 |
logs.append(
|
| 893 |
" | ".join(
|
| 894 |
f"{bucket_titles[key]}: {len(bucket_dates[key])}"
|
|
@@ -897,6 +905,7 @@ def fetch_bucketed_recent_activity(
|
|
| 897 |
)
|
| 898 |
|
| 899 |
result["logs"] = "\n".join(logs)
|
|
|
|
| 900 |
set_cache(cache_key, result)
|
| 901 |
return result
|
| 902 |
|
|
@@ -1735,7 +1744,12 @@ def render_invalid_app_state(app_state: dict | None):
|
|
| 1735 |
error_text = (app_state or {}).get("validation_error") or f"Username or org '{username}' was not found."
|
| 1736 |
|
| 1737 |
empty_counts = {kind: 0 for kind in REPO_KINDS}
|
| 1738 |
-
empty_metrics = render_metric_cards(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1739 |
|
| 1740 |
return (
|
| 1741 |
"## Ready",
|
|
@@ -1757,7 +1771,12 @@ def render_app_state(
|
|
| 1757 |
if not app_state or not app_state.get("username"):
|
| 1758 |
selected_year = datetime.now().year
|
| 1759 |
empty_counts = {kind: 0 for kind in REPO_KINDS}
|
| 1760 |
-
empty_metrics = render_metric_cards(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1761 |
context_html = render_idle_context_banner("", profile)
|
| 1762 |
return (
|
| 1763 |
"## Ready",
|
|
@@ -1915,11 +1934,7 @@ def fetch_and_render_repo_only(
|
|
| 1915 |
logs_box,
|
| 1916 |
)
|
| 1917 |
|
| 1918 |
-
requested_visibility =
|
| 1919 |
-
current_visibility_mode
|
| 1920 |
-
if current_visibility_mode in {"all", "public"}
|
| 1921 |
-
else app_state["default_visibility"]
|
| 1922 |
-
)
|
| 1923 |
final_visibility = requested_visibility if app_state["can_toggle_visibility"] else "public"
|
| 1924 |
|
| 1925 |
title_md, context_box, metrics_box, all_chart, model_chart, dataset_chart, space_chart, logs_box = render_app_state(
|
|
|
|
| 1 |
import html
|
|
|
|
| 2 |
import time
|
| 3 |
+
import threading
|
| 4 |
from datetime import datetime
|
| 5 |
from functools import lru_cache
|
| 6 |
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 7 |
|
| 8 |
import gradio as gr
|
| 9 |
import pandas as pd
|
|
|
|
| 33 |
|
| 34 |
RECENT_ACTIVITY_LIMIT = 20
|
| 35 |
RECENT_ACTIVITY_MAX_PAGES = 2000
|
|
|
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
# ============================================================
|
|
|
|
| 66 |
if isinstance(events, list):
|
| 67 |
return events, payload.get("cursor")
|
| 68 |
|
| 69 |
+
for key in ["items", "events", "activity"]:
|
| 70 |
value = payload.get(key)
|
| 71 |
if isinstance(value, list):
|
| 72 |
return value, payload.get("cursor")
|
|
|
|
| 80 |
return start_ts, end_ts
|
| 81 |
|
| 82 |
|
| 83 |
+
def should_stop_after_crossing_year_start(
|
| 84 |
+
crossed_before_year_start: bool,
|
|
|
|
| 85 |
) -> bool:
|
| 86 |
+
return crossed_before_year_start
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
|
| 89 |
# ============================================================
|
|
|
|
| 515 |
pages_fetched = 0
|
| 516 |
|
| 517 |
seen_target_year = False
|
| 518 |
+
crossed_before_year_start = False
|
| 519 |
oldest_event_ts_seen = None
|
| 520 |
|
| 521 |
+
OPEN_EVENT_EPSILON_SECONDS = 90
|
| 522 |
logs = []
|
| 523 |
+
|
| 524 |
+
year_start, year_end = get_year_bounds(selected_year)
|
| 525 |
session = build_session("application/json")
|
| 526 |
|
| 527 |
def extract_author_name(discussion: dict) -> str | None:
|
|
|
|
| 529 |
if isinstance(author, str):
|
| 530 |
return author
|
| 531 |
if isinstance(author, dict):
|
| 532 |
+
for key in ["name", "username", "login"]:
|
| 533 |
value = author.get(key)
|
| 534 |
if isinstance(value, str):
|
| 535 |
return value
|
| 536 |
return None
|
| 537 |
|
| 538 |
+
def parse_ts(raw_value):
|
| 539 |
+
if not raw_value:
|
| 540 |
+
return None
|
| 541 |
+
try:
|
| 542 |
+
return pd.to_datetime(raw_value, utc=True)
|
| 543 |
+
except Exception:
|
| 544 |
+
return None
|
| 545 |
+
|
| 546 |
def event_matches_mode(event: dict, discussion: dict) -> tuple[bool, str | None]:
|
| 547 |
is_pr = bool(discussion.get("isPullRequest", False))
|
| 548 |
author_name = extract_author_name(discussion)
|
|
|
|
| 577 |
return False, None
|
| 578 |
|
| 579 |
def extract_timestamp_for_mode(event: dict, discussion: dict):
|
| 580 |
+
event_ts = parse_ts(event.get("time"))
|
| 581 |
+
created_ts = parse_ts(discussion.get("createdAt"))
|
| 582 |
|
| 583 |
if mode in {"pr_opened", "thread_opened"}:
|
| 584 |
return created_ts
|
|
|
|
| 588 |
return None
|
| 589 |
if created_ts is not None:
|
| 590 |
delta_seconds = abs((event_ts - created_ts).total_seconds())
|
| 591 |
+
if delta_seconds <= OPEN_EVENT_EPSILON_SECONDS:
|
| 592 |
return None
|
| 593 |
return event_ts
|
| 594 |
|
|
|
|
| 616 |
response = session.get(
|
| 617 |
"https://huggingface.co/api/recent-activity",
|
| 618 |
params=params,
|
| 619 |
+
timeout=20,
|
| 620 |
)
|
| 621 |
response.raise_for_status()
|
| 622 |
|
|
|
|
| 633 |
if not isinstance(event, dict):
|
| 634 |
continue
|
| 635 |
|
| 636 |
+
event_ts = parse_ts(event.get("time"))
|
| 637 |
if event_ts is not None:
|
| 638 |
if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
|
| 639 |
oldest_event_ts_seen = event_ts
|
|
|
|
| 641 |
if year_start <= event_ts <= year_end:
|
| 642 |
seen_target_year = True
|
| 643 |
elif event_ts < year_start:
|
| 644 |
+
crossed_before_year_start = True
|
| 645 |
|
| 646 |
discussion = event.get("discussionData") or {}
|
| 647 |
if not isinstance(discussion, dict):
|
|
|
|
| 668 |
seen_ids.add(unique_id)
|
| 669 |
dates.append(dt.tz_localize(None).date())
|
| 670 |
|
| 671 |
+
if should_stop_after_crossing_year_start(crossed_before_year_start):
|
| 672 |
+
logs.append("Stopped after crossing before selected year.")
|
| 673 |
break
|
| 674 |
|
| 675 |
if next_cursor:
|
|
|
|
| 681 |
if oldest_event_ts_seen is not None:
|
| 682 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
| 683 |
logs.append(f"Seen target year: {seen_target_year}.")
|
| 684 |
+
logs.append(f"Crossed before year start: {crossed_before_year_start}.")
|
| 685 |
logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
|
| 686 |
|
| 687 |
return {
|
|
|
|
| 784 |
pages_fetched = 0
|
| 785 |
|
| 786 |
seen_target_year = False
|
| 787 |
+
crossed_before_year_start = False
|
| 788 |
oldest_event_ts_seen = None
|
| 789 |
|
|
|
|
| 790 |
logs = []
|
| 791 |
+
|
| 792 |
+
year_start, year_end = get_year_bounds(selected_year)
|
| 793 |
session = build_session("application/json")
|
| 794 |
|
| 795 |
try:
|
|
|
|
| 814 |
response = session.get(
|
| 815 |
"https://huggingface.co/api/recent-activity",
|
| 816 |
params=params,
|
| 817 |
+
timeout=20,
|
| 818 |
)
|
| 819 |
response.raise_for_status()
|
| 820 |
|
|
|
|
| 831 |
if not isinstance(event, dict):
|
| 832 |
continue
|
| 833 |
|
| 834 |
+
raw_time = event.get("time")
|
| 835 |
+
if not raw_time:
|
| 836 |
+
continue
|
| 837 |
+
|
| 838 |
+
try:
|
| 839 |
+
event_ts = pd.to_datetime(raw_time, utc=True)
|
| 840 |
+
except Exception:
|
| 841 |
continue
|
| 842 |
|
| 843 |
if oldest_event_ts_seen is None or event_ts < oldest_event_ts_seen:
|
|
|
|
| 846 |
if year_start <= event_ts <= year_end:
|
| 847 |
seen_target_year = True
|
| 848 |
elif event_ts < year_start:
|
| 849 |
+
crossed_before_year_start = True
|
| 850 |
|
| 851 |
if event_ts > year_end:
|
| 852 |
continue
|
| 853 |
+
|
| 854 |
if event_ts < year_start:
|
| 855 |
continue
|
| 856 |
|
| 857 |
bucket_key, unique_id = classifier(event)
|
| 858 |
+
|
| 859 |
if not bucket_key or not unique_id:
|
| 860 |
continue
|
| 861 |
+
|
| 862 |
if bucket_key not in bucket_dates:
|
| 863 |
continue
|
| 864 |
+
|
| 865 |
if unique_id in bucket_seen_ids[bucket_key]:
|
| 866 |
continue
|
| 867 |
|
| 868 |
bucket_seen_ids[bucket_key].add(unique_id)
|
| 869 |
bucket_dates[bucket_key].append(event_ts.tz_localize(None).date())
|
| 870 |
|
| 871 |
+
if should_stop_after_crossing_year_start(crossed_before_year_start):
|
| 872 |
+
logs.append("Stopped after crossing before selected year.")
|
| 873 |
break
|
| 874 |
|
| 875 |
if next_cursor:
|
|
|
|
| 896 |
if oldest_event_ts_seen is not None:
|
| 897 |
logs.append(f"Oldest event reached: {oldest_event_ts_seen.date()}.")
|
| 898 |
logs.append(f"Seen target year: {seen_target_year}.")
|
| 899 |
+
logs.append(f"Crossed before year start: {crossed_before_year_start}.")
|
| 900 |
logs.append(
|
| 901 |
" | ".join(
|
| 902 |
f"{bucket_titles[key]}: {len(bucket_dates[key])}"
|
|
|
|
| 905 |
)
|
| 906 |
|
| 907 |
result["logs"] = "\n".join(logs)
|
| 908 |
+
|
| 909 |
set_cache(cache_key, result)
|
| 910 |
return result
|
| 911 |
|
|
|
|
| 1744 |
error_text = (app_state or {}).get("validation_error") or f"Username or org '{username}' was not found."
|
| 1745 |
|
| 1746 |
empty_counts = {kind: 0 for kind in REPO_KINDS}
|
| 1747 |
+
empty_metrics = render_metric_cards(
|
| 1748 |
+
0,
|
| 1749 |
+
empty_counts,
|
| 1750 |
+
empty_counts,
|
| 1751 |
+
selected_year,
|
| 1752 |
+
)
|
| 1753 |
|
| 1754 |
return (
|
| 1755 |
"## Ready",
|
|
|
|
| 1771 |
if not app_state or not app_state.get("username"):
|
| 1772 |
selected_year = datetime.now().year
|
| 1773 |
empty_counts = {kind: 0 for kind in REPO_KINDS}
|
| 1774 |
+
empty_metrics = render_metric_cards(
|
| 1775 |
+
0,
|
| 1776 |
+
empty_counts,
|
| 1777 |
+
empty_counts,
|
| 1778 |
+
selected_year,
|
| 1779 |
+
)
|
| 1780 |
context_html = render_idle_context_banner("", profile)
|
| 1781 |
return (
|
| 1782 |
"## Ready",
|
|
|
|
| 1934 |
logs_box,
|
| 1935 |
)
|
| 1936 |
|
| 1937 |
+
requested_visibility = current_visibility_mode if current_visibility_mode in {"all", "public"} else app_state["default_visibility"]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1938 |
final_visibility = requested_visibility if app_state["can_toggle_visibility"] else "public"
|
| 1939 |
|
| 1940 |
title_md, context_box, metrics_box, all_chart, model_chart, dataset_chart, space_chart, logs_box = render_app_state(
|