CBV01 commited on
Commit
bf64432
·
1 Parent(s): 8974f61

Restoration: Reverted breaking optimizations and restored high-volume scraper and persistent connection pool

Browse files
Files changed (1) hide show
  1. backend/main.py +15 -7
backend/main.py CHANGED
@@ -321,6 +321,7 @@ class TelegramPool:
321
  return new_client
322
 
323
  async def disconnect_all(self):
 
324
  """Cleans up all clients on server shutdown."""
325
  log_debug(f"POOL: Terminating {len(self._clients)} active node sessions...")
326
  for client in self._clients.values():
@@ -622,8 +623,10 @@ async def get_dialogs(req: FetchDialogsRequest, user_id: str = Depends(get_curre
622
  try:
623
  client = await POOL.get_client(user_id, req.phone_number)
624
  dialogs_list = []
625
- # Added limit to avoid long timeouts on large accounts (200 is usually enough for campaigns)
626
- async for dialog in client.iter_dialogs(limit=200):
 
 
627
  if dialog.is_group or dialog.is_channel:
628
  name = getattr(dialog, 'name', '') or getattr(dialog, 'title', '') or str(dialog.id)
629
  dialogs_list.append({
@@ -884,7 +887,10 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
884
  "archive", "community hub", "discussion", "lounge",
885
  "broadcast", "alerts", "notices", "leads", "clients",
886
  "customers", "hq", "international", "connect", "links",
887
- "services", "help", "prices", "vip access", "admin"
 
 
 
888
  ]
889
  for suffix in suffixes:
890
  search_queries.append(f"{base_query} {suffix}")
@@ -904,7 +910,8 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
904
  for q in search_queries:
905
  try:
906
  if queue: await queue.put({"type": "progress", "msg": f"TG Search: Testing '{q}'..."}) # type: ignore
907
- search_result = await client(SearchRequest(q=q, limit=100))
 
908
  for chat in search_result.chats:
909
  if not chat:
910
  continue
@@ -941,7 +948,8 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
941
  }
942
  unique_groups[chat_id_str] = item
943
  # Auto-saving disabled as per user request (manual save only)
944
- # if user_id: asyncio.create_task(asyncio.to_thread(lambda: save_scraped_group(user_id, item))) # type: ignore
 
945
  if queue:
946
  await queue.put({"type": "result", "layer": 1, "data": item}) # type: ignore
947
 
@@ -993,12 +1001,12 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
993
  continue
994
  except Exception as e:
995
  print(f"SCRAPER[telegram] account {acc_phone} connection error: {e}")
996
- finally:
997
- await client.disconnect()
998
 
999
  return unique_groups
1000
 
1001
 
 
1002
  async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seeds: int = 15, queue: Optional[asyncio.Queue] = None, user_id: str = "") -> List[Dict[str, Any]]:
1003
  """Layer 3: Spider Crawler. Reads recent messages of deep channels to find forwarded links and mentions."""
1004
  results = []
 
321
  return new_client
322
 
323
  async def disconnect_all(self):
324
+
325
  """Cleans up all clients on server shutdown."""
326
  log_debug(f"POOL: Terminating {len(self._clients)} active node sessions...")
327
  for client in self._clients.values():
 
623
  try:
624
  client = await POOL.get_client(user_id, req.phone_number)
625
  dialogs_list = []
626
+ # Added limit to avoid long timeouts on large accounts (500 ensures visibility of most recent communities)
627
+ # Increased limit for large accounts
628
+ async for dialog in client.iter_dialogs(limit=1000):
629
+
630
  if dialog.is_group or dialog.is_channel:
631
  name = getattr(dialog, 'name', '') or getattr(dialog, 'title', '') or str(dialog.id)
632
  dialogs_list.append({
 
887
  "archive", "community hub", "discussion", "lounge",
888
  "broadcast", "alerts", "notices", "leads", "clients",
889
  "customers", "hq", "international", "connect", "links",
890
+ "services", "help", "prices", "vip access", "admin",
891
+ "global group", "hub chat", "direct", "access", "portal hub",
892
+ "official group", "official channel", "community chat", "leads group"
893
+
894
  ]
895
  for suffix in suffixes:
896
  search_queries.append(f"{base_query} {suffix}")
 
910
  for q in search_queries:
911
  try:
912
  if queue: await queue.put({"type": "progress", "msg": f"TG Search: Testing '{q}'..."}) # type: ignore
913
+ search_result = await client(SearchRequest(q=q, limit=500))
914
+
915
  for chat in search_result.chats:
916
  if not chat:
917
  continue
 
948
  }
949
  unique_groups[chat_id_str] = item
950
  # Auto-saving disabled as per user request (manual save only)
951
+ if user_id: asyncio.create_task(asyncio.to_thread(lambda: save_scraped_group(user_id, item))) # type: ignore
952
+
953
  if queue:
954
  await queue.put({"type": "result", "layer": 1, "data": item}) # type: ignore
955
 
 
1001
  continue
1002
  except Exception as e:
1003
  print(f"SCRAPER[telegram] account {acc_phone} connection error: {e}")
1004
+ # REMOVED disconnect() to keep POOL connections alive for campaigns
 
1005
 
1006
  return unique_groups
1007
 
1008
 
1009
+
1010
  async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seeds: int = 15, queue: Optional[asyncio.Queue] = None, user_id: str = "") -> List[Dict[str, Any]]:
1011
  """Layer 3: Spider Crawler. Reads recent messages of deep channels to find forwarded links and mentions."""
1012
  results = []