lb-autoheal: keep A's history fresh + cheap history-only re-seed for deltas

Proxy mode had disabled A's maintenance worker AND its 600MB save, which
is also where the hourly/daily history snapshots were taken -- so A stopped
producing history and could no longer seed the mirror's d1/d7 deltas after
a mirror restart. Two fixes in the auto-heal loop:

- _sm_lb_keep_history_fresh(): re-take the cheap, index-independent hourly
+ daily snapshots each tick so A stays the durable bootstrap source.
- history-only re-seed: when the mirror has rows but no d1 baseline (fresh
restart), push just the history files -- no 1.6M row re-push, no
starvation risk. The full mirror sync now also pushes history FIRST, so a
heal interrupted mid-rows still lands the deltas.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app.py

@@ -12494,6 +12494,28 @@ def _sm_lb_proxy_fetch(request: Request, path: str, inject_counts: bool = False)
 _SM_LB_MIRROR_STATUS = {"running": False, "pushed_rows": 0, "pushed_hist": 0, "total_rows": 0,
                         "done": False, "error": ""}
 
+def _sm_lb_push_history(target: str, sess, hdr, st: dict = None):
+    """Push A's daily + hourly history snapshots to the mirror (immediate d1/d7 deltas). Small +
+    quick; done BEFORE the heavy row push so a heal interrupted mid-rows still lands the deltas."""
+    n = 0
+    import gzip as _gz
+    for f in sorted(SM_LB_HISTORY_DIR.glob("*.json")):
+        data = json.loads(f.read_text(encoding="utf-8"))
+        sess.post(f"{target}/lb-history-import",
+                  json={"kind": "daily", "name": f.stem, "data": data}, headers=hdr, timeout=90).raise_for_status()
+        n += 1
+        if st is not None:
+            st["pushed_hist"] = n
+    for f in sorted(SM_LB_HISTORY_HOURLY_DIR.glob("*.json.gz")):
+        data = json.loads(_gz.decompress(f.read_bytes()).decode("utf-8"))
+        sess.post(f"{target}/lb-history-import",
+                  json={"kind": "hourly", "name": f.name[:-len('.json.gz')], "data": data},
+                  headers=hdr, timeout=90).raise_for_status()
+        n += 1
+        if st is not None:
+            st["pushed_hist"] = n
+    return n
+
 def _sm_lb_mirror_sync_worker(target: str, chunk: int):
     st = _SM_LB_MIRROR_STATUS
     st.update({"running": True, "pushed_rows": 0, "pushed_hist": 0, "total_rows": 0,
@@ -12501,6 +12523,12 @@ def _sm_lb_mirror_sync_worker(target: str, chunk: int):
     sess = _requests.Session()
     hdr = {"X-Ingest-Key": SM_LEADERBOARD_INGEST_KEY, "Content-Type": "application/json"}
     try:
+        # history FIRST: it's tiny and gives deltas immediately; if the long row push below is
+        # interrupted (e.g. A gets restarted under the load) the mirror still has its d1/d7.
+        try:
+            _sm_lb_push_history(target, sess, hdr, st)
+        except Exception as he:  # noqa: BLE001 -- best-effort; rows still follow
+            print(f"[lb-mirror-sync] history push warning: {type(he).__name__}: {str(he)[:150]}")
         with _SM_LEADERBOARD_LOCK:
             rows_src = list((_sm_leaderboard_load_unlocked().get("players") or {}).values())
         st["total_rows"] = len(rows_src)
@@ -12532,22 +12560,6 @@ def _sm_lb_mirror_sync_worker(target: str, chunk: int):
                 flush()
                 time.sleep(0.3)          # gentle: don't hammer the mirror during the one-time sync
         flush()
-        # history snapshots -> immediate deltas on the mirror
-        try:
-            import gzip as _gz
-            for f in sorted(SM_LB_HISTORY_DIR.glob("*.json")):
-                data = json.loads(f.read_text(encoding="utf-8"))
-                sess.post(f"{target}/lb-history-import",
-                          json={"kind": "daily", "name": f.stem, "data": data}, headers=hdr, timeout=90).raise_for_status()
-                st["pushed_hist"] += 1
-            for f in sorted(SM_LB_HISTORY_HOURLY_DIR.glob("*.json.gz")):
-                data = json.loads(_gz.decompress(f.read_bytes()).decode("utf-8"))
-                sess.post(f"{target}/lb-history-import",
-                          json={"kind": "hourly", "name": f.name[:-len('.json.gz')], "data": data},
-                          headers=hdr, timeout=90).raise_for_status()
-                st["pushed_hist"] += 1
-        except Exception as he:  # noqa: BLE001 -- history is best-effort; accounts already synced
-            print(f"[lb-mirror-sync] history push warning: {type(he).__name__}: {str(he)[:150]}")
         st["done"] = True
         print(f"[lb-mirror-sync] done: {st['pushed_rows']}/{st['total_rows']} rows, "
               f"{st['pushed_hist']} history files -> {target}")
@@ -12618,13 +12630,43 @@ def _sm_lb_mirror_stored_count():
     except Exception:  # noqa: BLE001
         return 0, False
 
+def _sm_lb_mirror_missing_d1():
+    """True if the mirror is up but has no ~24h-ago snapshot (d1 deltas blank). A fresh mirror
+    (post-restart, before it accumulates its own hourly snapshots) has this until A re-seeds its
+    history -- cheap to fix without re-pushing 1.6M rows."""
+    try:
+        global _SM_LB_PROXY_SESSION
+        if _SM_LB_PROXY_SESSION is None:
+            _SM_LB_PROXY_SESSION = _requests.Session()
+        r = _SM_LB_PROXY_SESSION.get(f"{SM_LEADERBOARD_PROXY_URL}/leaderboard?limit=1&measure=d1", timeout=8)
+        r.raise_for_status()
+        return (r.json() or {}).get("history", {}).get("d1_age") is None
+    except Exception:  # noqa: BLE001
+        return False   # unreachable -> not our concern here (the thin/serving gates handle it)
+
+def _sm_lb_keep_history_fresh():
+    """Proxy mode disables A's heavy maintenance worker AND its 600MB save -- which is also where
+    the hourly/daily history snapshots were taken. So A stops producing history and, days later,
+    can no longer seed the mirror's d1/d7 after a restart. Re-take the (cheap, index-independent)
+    snapshots here so A always holds fresh bootstrap history. Once/hour + once/day of real work;
+    a file-exists check otherwise."""
+    try:
+        _sm_leaderboard_snapshot_hourly()
+        with _SM_LEADERBOARD_LOCK:
+            state = _sm_leaderboard_load_unlocked()
+        _sm_leaderboard_snapshot_daily(state)
+    except Exception as exc:  # noqa: BLE001
+        print(f"[lb-autoheal] snapshot keep-fresh: {type(exc).__name__}: {str(exc)[:120]}")
+
 def _sm_lb_autoheal_worker():
     if not (SM_LEADERBOARD_PROXY_URL and SM_LEADERBOARD_INGEST_KEY):
         return
     time.sleep(75)   # let boot + the pusher's first sweep settle before judging the mirror
     last_sync = 0.0
+    last_hist = 0.0
     while True:
         try:
+            _sm_lb_keep_history_fresh()   # A stays the durable source of bootstrap history
             if not _SM_LB_MIRROR_STATUS.get("running"):
                 mirror_rows, reachable = _sm_lb_mirror_stored_count()
                 local_rows = _sm_lb_local_stored_count()
@@ -12639,6 +12681,19 @@ def _sm_lb_autoheal_worker():
                     _sm_lb_mirror_sync_worker(SM_LEADERBOARD_PROXY_URL, 5000)
                     last_sync = time.time()
                     _sm_lb_proxy_healthy(force=True)   # refresh the serving gate
+                elif reachable and not thin and (time.time() - last_hist) > SM_LB_AUTOHEAL_COOLDOWN \
+                        and _sm_lb_mirror_missing_d1():
+                    # Rows are fine but the mirror has no d1 baseline (fresh restart): re-seed just
+                    # the history -- tiny, no 1.6M row re-push, no starvation risk.
+                    print("[lb-autoheal] mirror missing d1 history -> re-seeding history only")
+                    try:
+                        _sess = _requests.Session()
+                        _hdr = {"X-Ingest-Key": SM_LEADERBOARD_INGEST_KEY, "Content-Type": "application/json"}
+                        n = _sm_lb_push_history(SM_LEADERBOARD_PROXY_URL, _sess, _hdr)
+                        print(f"[lb-autoheal] re-seeded {n} history files")
+                    except Exception as he:  # noqa: BLE001
+                        print(f"[lb-autoheal] history re-seed failed: {type(he).__name__}: {str(he)[:120]}")
+                    last_hist = time.time()
         except Exception as exc:  # noqa: BLE001
             print(f"[lb-autoheal] {type(exc).__name__}: {str(exc)[:150]}")
         time.sleep(SM_LB_AUTOHEAL_INTERVAL)