Spaces:
Paused
Paused
CBV01 commited on
Commit ·
39e1e78
1
Parent(s): 7b1f500
Turbocharged Scraper: merging results from all user accounts for unlimited reach
Browse files- backend/main.py +62 -48
backend/main.py
CHANGED
|
@@ -360,61 +360,75 @@ async def get_accounts(user_id: str = Depends(get_current_user_id)):
|
|
| 360 |
return {"accounts": accounts}
|
| 361 |
|
| 362 |
@app.get("/api/telegram/scrape")
|
| 363 |
-
async def scrape_keyword(query: str, limit: int =
|
| 364 |
conn = get_db_connection()
|
| 365 |
if phone_number:
|
| 366 |
-
|
|
|
|
| 367 |
else:
|
| 368 |
-
|
| 369 |
-
|
| 370 |
|
| 371 |
-
|
| 372 |
-
client = TelegramClient(StringSession(session_str), api_id, api_hash)
|
| 373 |
-
await client.connect()
|
| 374 |
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
if not
|
| 385 |
-
|
| 386 |
|
| 387 |
-
#
|
| 388 |
-
|
| 389 |
-
title_str = str(getattr(chat, 'title', '')).lower()
|
| 390 |
-
countries = {"nigeria": "NG", "usa": "US", "uk": "UK", "india": "IN", "crypto": "Global"}
|
| 391 |
-
for k, v in countries.items():
|
| 392 |
-
if k in title_str: country = v; break
|
| 393 |
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
|
| 419 |
class BulkJoinRequest(BaseModel):
|
| 420 |
group_ids: list[str]
|
|
|
|
| 360 |
return {"accounts": accounts}
|
| 361 |
|
| 362 |
@app.get("/api/telegram/scrape")
|
| 363 |
+
async def scrape_keyword(query: str, limit: int = 1000, phone_number: Optional[str] = None, user_id: str = Depends(get_current_user_id)):
|
| 364 |
conn = get_db_connection()
|
| 365 |
if phone_number:
|
| 366 |
+
# Use specific account
|
| 367 |
+
rows = conn.execute("SELECT session_string, api_id, api_hash, phone_number FROM accounts WHERE phone_number = ? AND user_id = ?", (phone_number, user_id)).fetchall()
|
| 368 |
else:
|
| 369 |
+
# Use ALL accounts to maximize results
|
| 370 |
+
rows = conn.execute("SELECT session_string, api_id, api_hash, phone_number FROM accounts WHERE user_id = ? AND status = 'active'", (user_id,)).fetchall()
|
| 371 |
|
| 372 |
+
if not rows: raise HTTPException(status_code=404, detail="No active accounts found.")
|
|
|
|
|
|
|
| 373 |
|
| 374 |
+
unique_groups: Dict[str, Dict[str, Any]] = {}
|
| 375 |
+
|
| 376 |
+
# Iterate through accounts to get as many results as possible
|
| 377 |
+
# Telegram results vary by account/region/trust
|
| 378 |
+
for row in rows:
|
| 379 |
+
session_str, api_id, api_hash, acc_phone = row[0], int(row[1]), row[2], row[3]
|
| 380 |
+
client = TelegramClient(StringSession(session_str), api_id, api_hash)
|
| 381 |
+
try:
|
| 382 |
+
await client.connect()
|
| 383 |
+
if not await client.is_user_authorized():
|
| 384 |
+
continue
|
| 385 |
|
| 386 |
+
# Requesting high limit. Merge with what we already have.
|
| 387 |
+
search_result = await client(SearchRequest(q=query, limit=limit))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
|
| 389 |
+
for chat in search_result.chats:
|
| 390 |
+
# We skip deleted chats but take everything else (even min objects)
|
| 391 |
+
if not chat: continue
|
| 392 |
+
|
| 393 |
+
chat_id_str = str(getattr(chat, 'id', '0'))
|
| 394 |
+
if chat_id_str in unique_groups: continue
|
| 395 |
+
|
| 396 |
+
is_private = getattr(chat, 'username', None) is None
|
| 397 |
+
|
| 398 |
+
title = getattr(chat, 'title', 'Unknown')
|
| 399 |
+
username = getattr(chat, 'username', None)
|
| 400 |
+
|
| 401 |
+
# Metadata extraction
|
| 402 |
+
participants_count = getattr(chat, 'participants_count', 0) or getattr(chat, 'megagroup_participants', 0) or 0
|
| 403 |
+
chat_type = 'channel' if getattr(chat, 'broadcast', False) else 'group'
|
| 404 |
+
|
| 405 |
+
# Check history
|
| 406 |
+
conn2 = get_db_connection()
|
| 407 |
+
# Isolation: Ensure we track this for the specific user
|
| 408 |
+
conn2.execute("INSERT OR IGNORE INTO scrape_history (group_id, user_id) VALUES (?, ?)", (chat_id_str, user_id))
|
| 409 |
+
if hasattr(conn2, "commit"): conn2.commit()
|
| 410 |
+
global_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ?", (chat_id_str,)).fetchone()[0]
|
| 411 |
+
user_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ? AND user_id = ?", (chat_id_str, user_id)).fetchone()[0]
|
| 412 |
+
if hasattr(conn2, "close"): conn2.close()
|
| 413 |
+
|
| 414 |
+
unique_groups[chat_id_str] = {
|
| 415 |
+
"id": chat_id_str,
|
| 416 |
+
"title": title,
|
| 417 |
+
"username": username,
|
| 418 |
+
"participants_count": participants_count,
|
| 419 |
+
"type": chat_type,
|
| 420 |
+
"is_private": is_private,
|
| 421 |
+
"country": "Global",
|
| 422 |
+
"global_shows": global_count,
|
| 423 |
+
"user_shows": user_count
|
| 424 |
+
}
|
| 425 |
+
except Exception as e:
|
| 426 |
+
print(f"SCRAPER: Account {acc_phone} error: {e}")
|
| 427 |
+
finally:
|
| 428 |
+
await client.disconnect()
|
| 429 |
+
|
| 430 |
+
if hasattr(conn, "close"): conn.close()
|
| 431 |
+
return {"groups": list(unique_groups.values())}
|
| 432 |
|
| 433 |
class BulkJoinRequest(BaseModel):
|
| 434 |
group_ids: list[str]
|