Spaces:
Paused
Paused
CBV01 commited on
Commit ·
bf64432
1
Parent(s): 8974f61
Restoration: Reverted breaking optimizations and restored high-volume scraper and persistent connection pool
Browse files- backend/main.py +15 -7
backend/main.py
CHANGED
|
@@ -321,6 +321,7 @@ class TelegramPool:
|
|
| 321 |
return new_client
|
| 322 |
|
| 323 |
async def disconnect_all(self):
|
|
|
|
| 324 |
"""Cleans up all clients on server shutdown."""
|
| 325 |
log_debug(f"POOL: Terminating {len(self._clients)} active node sessions...")
|
| 326 |
for client in self._clients.values():
|
|
@@ -622,8 +623,10 @@ async def get_dialogs(req: FetchDialogsRequest, user_id: str = Depends(get_curre
|
|
| 622 |
try:
|
| 623 |
client = await POOL.get_client(user_id, req.phone_number)
|
| 624 |
dialogs_list = []
|
| 625 |
-
# Added limit to avoid long timeouts on large accounts (
|
| 626 |
-
|
|
|
|
|
|
|
| 627 |
if dialog.is_group or dialog.is_channel:
|
| 628 |
name = getattr(dialog, 'name', '') or getattr(dialog, 'title', '') or str(dialog.id)
|
| 629 |
dialogs_list.append({
|
|
@@ -884,7 +887,10 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
|
|
| 884 |
"archive", "community hub", "discussion", "lounge",
|
| 885 |
"broadcast", "alerts", "notices", "leads", "clients",
|
| 886 |
"customers", "hq", "international", "connect", "links",
|
| 887 |
-
"services", "help", "prices", "vip access", "admin"
|
|
|
|
|
|
|
|
|
|
| 888 |
]
|
| 889 |
for suffix in suffixes:
|
| 890 |
search_queries.append(f"{base_query} {suffix}")
|
|
@@ -904,7 +910,8 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
|
|
| 904 |
for q in search_queries:
|
| 905 |
try:
|
| 906 |
if queue: await queue.put({"type": "progress", "msg": f"TG Search: Testing '{q}'..."}) # type: ignore
|
| 907 |
-
search_result = await client(SearchRequest(q=q, limit=
|
|
|
|
| 908 |
for chat in search_result.chats:
|
| 909 |
if not chat:
|
| 910 |
continue
|
|
@@ -941,7 +948,8 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
|
|
| 941 |
}
|
| 942 |
unique_groups[chat_id_str] = item
|
| 943 |
# Auto-saving disabled as per user request (manual save only)
|
| 944 |
-
|
|
|
|
| 945 |
if queue:
|
| 946 |
await queue.put({"type": "result", "layer": 1, "data": item}) # type: ignore
|
| 947 |
|
|
@@ -993,12 +1001,12 @@ async def _scrape_telegram_search(base_query: str, rows: list, queue: Optional[a
|
|
| 993 |
continue
|
| 994 |
except Exception as e:
|
| 995 |
print(f"SCRAPER[telegram] account {acc_phone} connection error: {e}")
|
| 996 |
-
|
| 997 |
-
await client.disconnect()
|
| 998 |
|
| 999 |
return unique_groups
|
| 1000 |
|
| 1001 |
|
|
|
|
| 1002 |
async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seeds: int = 15, queue: Optional[asyncio.Queue] = None, user_id: str = "") -> List[Dict[str, Any]]:
|
| 1003 |
"""Layer 3: Spider Crawler. Reads recent messages of deep channels to find forwarded links and mentions."""
|
| 1004 |
results = []
|
|
|
|
| 321 |
return new_client
|
| 322 |
|
| 323 |
async def disconnect_all(self):
|
| 324 |
+
|
| 325 |
"""Cleans up all clients on server shutdown."""
|
| 326 |
log_debug(f"POOL: Terminating {len(self._clients)} active node sessions...")
|
| 327 |
for client in self._clients.values():
|
|
|
|
| 623 |
try:
|
| 624 |
client = await POOL.get_client(user_id, req.phone_number)
|
| 625 |
dialogs_list = []
|
| 626 |
+
# Added limit to avoid long timeouts on large accounts (500 ensures visibility of most recent communities)
|
| 627 |
+
# Increased limit for large accounts
|
| 628 |
+
async for dialog in client.iter_dialogs(limit=1000):
|
| 629 |
+
|
| 630 |
if dialog.is_group or dialog.is_channel:
|
| 631 |
name = getattr(dialog, 'name', '') or getattr(dialog, 'title', '') or str(dialog.id)
|
| 632 |
dialogs_list.append({
|
|
|
|
| 887 |
"archive", "community hub", "discussion", "lounge",
|
| 888 |
"broadcast", "alerts", "notices", "leads", "clients",
|
| 889 |
"customers", "hq", "international", "connect", "links",
|
| 890 |
+
"services", "help", "prices", "vip access", "admin",
|
| 891 |
+
"global group", "hub chat", "direct", "access", "portal hub",
|
| 892 |
+
"official group", "official channel", "community chat", "leads group"
|
| 893 |
+
|
| 894 |
]
|
| 895 |
for suffix in suffixes:
|
| 896 |
search_queries.append(f"{base_query} {suffix}")
|
|
|
|
| 910 |
for q in search_queries:
|
| 911 |
try:
|
| 912 |
if queue: await queue.put({"type": "progress", "msg": f"TG Search: Testing '{q}'..."}) # type: ignore
|
| 913 |
+
search_result = await client(SearchRequest(q=q, limit=500))
|
| 914 |
+
|
| 915 |
for chat in search_result.chats:
|
| 916 |
if not chat:
|
| 917 |
continue
|
|
|
|
| 948 |
}
|
| 949 |
unique_groups[chat_id_str] = item
|
| 950 |
# Auto-saving disabled as per user request (manual save only)
|
| 951 |
+
if user_id: asyncio.create_task(asyncio.to_thread(lambda: save_scraped_group(user_id, item))) # type: ignore
|
| 952 |
+
|
| 953 |
if queue:
|
| 954 |
await queue.put({"type": "result", "layer": 1, "data": item}) # type: ignore
|
| 955 |
|
|
|
|
| 1001 |
continue
|
| 1002 |
except Exception as e:
|
| 1003 |
print(f"SCRAPER[telegram] account {acc_phone} connection error: {e}")
|
| 1004 |
+
# REMOVED disconnect() to keep POOL connections alive for campaigns
|
|
|
|
| 1005 |
|
| 1006 |
return unique_groups
|
| 1007 |
|
| 1008 |
|
| 1009 |
+
|
| 1010 |
async def _scrape_spider(seed_groups: List[Dict[str, Any]], rows: list, max_seeds: int = 15, queue: Optional[asyncio.Queue] = None, user_id: str = "") -> List[Dict[str, Any]]:
|
| 1011 |
"""Layer 3: Spider Crawler. Reads recent messages of deep channels to find forwarded links and mentions."""
|
| 1012 |
results = []
|