fix: reduce false empty years in activity (deeper scan + better parsing)

app.py

@@ -31,6 +31,10 @@ REPO_KIND_PALETTES = {
     "space": "orange",
 }
 
+RECENT_ACTIVITY_LIMIT = 20
+RECENT_ACTIVITY_MAX_PAGES = 200
+RECENT_ACTIVITY_TIMEOUT_SECONDS = 45
+
 
 # ============================================================
 # API
@@ -54,6 +58,23 @@ def build_session(accept: str) -> requests.Session:
     return session
 
 
+def normalize_recent_activity_payload(payload):
+    if isinstance(payload, list):
+        return payload, None
+
+    if isinstance(payload, dict):
+        events = payload.get("recentActivity")
+        if isinstance(events, list):
+            return events, payload.get("cursor")
+
+        for key in ["items", "events", "activity"]:
+            value = payload.get(key)
+            if isinstance(value, list):
+                return value, payload.get("cursor")
+
+    return [], None
+
+
 # ============================================================
 # Thread-safe rate limiters
 # ============================================================
@@ -478,12 +499,14 @@ def fetch_user_discussion_activity(
     dates = []
     seen_ids = set()
     skip = 0
-    limit = 20
+    limit = RECENT_ACTIVITY_LIMIT
     cursor = None
     pages_fetched = 0
+    reached_selected_year = False
+    oldest_event_year_seen = None
 
-    MAX_PAGES = 40
-    MAX_DURATION = 25
+    MAX_PAGES = RECENT_ACTIVITY_MAX_PAGES
+    MAX_DURATION = RECENT_ACTIVITY_TIMEOUT_SECONDS
     OPEN_EVENT_EPSILON_SECONDS = 90
     start = time.time()
 
@@ -491,22 +514,6 @@ def fetch_user_discussion_activity(
 
     session = build_session("application/json")
 
-    def normalize_payload(payload):
-        if isinstance(payload, list):
-            return payload, None
-
-        if isinstance(payload, dict):
-            events = payload.get("recentActivity")
-            if isinstance(events, list):
-                return events, payload.get("cursor")
-
-            for key in ["items", "events", "activity"]:
-                value = payload.get(key)
-                if isinstance(value, list):
-                    return value, payload.get("cursor")
-
-        return [], None
-
     def extract_author_name(discussion: dict) -> str | None:
         author = discussion.get("author")
         if isinstance(author, str):
@@ -612,7 +619,7 @@ def fetch_user_discussion_activity(
             payload = response.json()
             pages_fetched += 1
 
-            events, next_cursor = normalize_payload(payload)
+            events, next_cursor = normalize_recent_activity_payload(payload)
 
             if not events:
                 logs.append("No more events.")
@@ -625,9 +632,17 @@ def fetch_user_discussion_activity(
                     continue
 
                 event_ts = parse_ts(event.get("time"))
-                if event_ts is not None and event_ts.year < selected_year:
-                    stop_early = True
-                    break
+                if event_ts is not None:
+                    event_year = int(event_ts.year)
+                    if oldest_event_year_seen is None or event_year < oldest_event_year_seen:
+                        oldest_event_year_seen = event_year
+
+                    if event_year == selected_year:
+                        reached_selected_year = True
+
+                    if event_year < selected_year:
+                        stop_early = True
+                        break
 
                 discussion = event.get("discussionData") or {}
                 if not isinstance(discussion, dict):
@@ -638,7 +653,12 @@ def fetch_user_discussion_activity(
                     continue
 
                 dt = extract_date_for_mode(event, discussion)
-                if dt is None or dt.year != selected_year:
+                if dt is None:
+                    continue
+
+                if dt.year == selected_year:
+                    reached_selected_year = True
+                if dt.year != selected_year:
                     continue
 
                 if unique_id in seen_ids:
@@ -657,6 +677,12 @@ def fetch_user_discussion_activity(
                 skip += limit
 
         logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
+        if oldest_event_year_seen is not None:
+            logs.append(f"Oldest event year reached in feed: {oldest_event_year_seen}.")
+        if reached_selected_year:
+            logs.append(f"Selected year {selected_year} was reached in the feed.")
+        else:
+            logs.append(f"Selected year {selected_year} was NOT reached in the feed.")
         logs.append(f"Found {len(dates)} unique {mode} event(s) in {selected_year}.")
 
         return {
@@ -755,11 +781,13 @@ def fetch_bucketed_recent_activity(
 
     cursor = None
     skip = 0
-    limit = 20
+    limit = RECENT_ACTIVITY_LIMIT
     pages_fetched = 0
+    reached_selected_year = False
+    oldest_event_year_seen = None
 
-    MAX_PAGES = 40
-    MAX_DURATION = 20
+    MAX_PAGES = RECENT_ACTIVITY_MAX_PAGES
+    MAX_DURATION = RECENT_ACTIVITY_TIMEOUT_SECONDS
     start = time.time()
 
     logs = []
@@ -799,8 +827,7 @@ def fetch_bucketed_recent_activity(
             payload = response.json()
             pages_fetched += 1
 
-            events = payload.get("recentActivity", [])
-            next_cursor = payload.get("cursor")
+            events, next_cursor = normalize_recent_activity_payload(payload)
 
             if not events:
                 logs.append("No more events.")
@@ -821,11 +848,18 @@ def fetch_bucketed_recent_activity(
                 except Exception:
                     continue
 
-                if event_ts.year < selected_year:
+                event_year = int(event_ts.year)
+                if oldest_event_year_seen is None or event_year < oldest_event_year_seen:
+                    oldest_event_year_seen = event_year
+
+                if event_year == selected_year:
+                    reached_selected_year = True
+
+                if event_year < selected_year:
                     stop_early = True
                     break
 
-                if event_ts.year != selected_year:
+                if event_year != selected_year:
                     continue
 
                 bucket_key, unique_id = classifier(event)
@@ -867,6 +901,12 @@ def fetch_bucketed_recent_activity(
             )
 
         logs.append(f"Fetched {pages_fetched} page(s) from recent-activity.")
+        if oldest_event_year_seen is not None:
+            logs.append(f"Oldest event year reached in feed: {oldest_event_year_seen}.")
+        if reached_selected_year:
+            logs.append(f"Selected year {selected_year} was reached in the feed.")
+        else:
+            logs.append(f"Selected year {selected_year} was NOT reached in the feed.")
         logs.append(
             " | ".join(
                 f"{bucket_titles[key]}: {len(bucket_dates[key])}"