CBV01 commited on
Commit
39e1e78
·
1 Parent(s): 7b1f500

Turbocharged Scraper: merging results from all user accounts for unlimited reach

Browse files
Files changed (1) hide show
  1. backend/main.py +62 -48
backend/main.py CHANGED
@@ -360,61 +360,75 @@ async def get_accounts(user_id: str = Depends(get_current_user_id)):
360
  return {"accounts": accounts}
361
 
362
  @app.get("/api/telegram/scrape")
363
- async def scrape_keyword(query: str, limit: int = 500, phone_number: Optional[str] = None, user_id: str = Depends(get_current_user_id)):
364
  conn = get_db_connection()
365
  if phone_number:
366
- row = conn.execute("SELECT session_string, api_id, api_hash FROM accounts WHERE phone_number = ? AND user_id = ?", (phone_number, user_id)).fetchone()
 
367
  else:
368
- row = conn.execute("SELECT session_string, api_id, api_hash FROM accounts WHERE user_id = ? LIMIT 1", (user_id,)).fetchone()
369
- if not row: raise HTTPException(status_code=404, detail="No active accounts found.")
370
 
371
- session_str, api_id, api_hash = row[0], int(row[1]), row[2]
372
- client = TelegramClient(StringSession(session_str), api_id, api_hash)
373
- await client.connect()
374
 
375
- groups: List[Dict[str, Any]] = []
376
- try:
377
- # Requesting a very high limit. Telegram usually caps Global Search at ~200-300 results,
378
- # but Telethon will try to paginate if possible or we can at least ensure we take everything we get.
379
- search_result = await client(SearchRequest(q=query, limit=limit))
380
-
381
- conn2 = get_db_connection()
382
- for chat in search_result.chats:
383
- # Skip deleted or restricted chats
384
- if not chat or getattr(chat, 'min', False): continue
385
- is_private = getattr(chat, 'username', None) is None
386
 
387
- # Simple country detection based on language/keywords or just "Global"
388
- country = "Global"
389
- title_str = str(getattr(chat, 'title', '')).lower()
390
- countries = {"nigeria": "NG", "usa": "US", "uk": "UK", "india": "IN", "crypto": "Global"}
391
- for k, v in countries.items():
392
- if k in title_str: country = v; break
393
 
394
- chat_id_str = str(getattr(chat, 'id', '0'))
395
- conn2.execute("INSERT INTO scrape_history (group_id, user_id) VALUES (?, ?)", (chat_id_str, user_id))
396
- if hasattr(conn2, "commit"): conn2.commit()
397
-
398
- global_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ?", (chat_id_str,)).fetchone()[0]
399
- user_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ? AND user_id = ?", (chat_id_str, user_id)).fetchone()[0]
400
-
401
- groups.append({
402
- "id": chat_id_str,
403
- "title": getattr(chat, 'title', 'Unknown'),
404
- "username": getattr(chat, 'username', None),
405
- "participants_count": getattr(chat, 'participants_count', 0) or getattr(chat, 'megagroup_participants', 0),
406
- "type": 'channel' if getattr(chat, 'broadcast', False) else 'group',
407
- "is_private": is_private,
408
- "country": country,
409
- "global_shows": global_count,
410
- "user_shows": user_count
411
- })
412
- if hasattr(conn2, "close"): conn2.close()
413
- return {"groups": groups}
414
- except Exception as e:
415
- raise HTTPException(status_code=400, detail=str(e))
416
- finally:
417
- await client.disconnect()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
 
419
  class BulkJoinRequest(BaseModel):
420
  group_ids: list[str]
 
360
  return {"accounts": accounts}
361
 
362
  @app.get("/api/telegram/scrape")
363
+ async def scrape_keyword(query: str, limit: int = 1000, phone_number: Optional[str] = None, user_id: str = Depends(get_current_user_id)):
364
  conn = get_db_connection()
365
  if phone_number:
366
+ # Use specific account
367
+ rows = conn.execute("SELECT session_string, api_id, api_hash, phone_number FROM accounts WHERE phone_number = ? AND user_id = ?", (phone_number, user_id)).fetchall()
368
  else:
369
+ # Use ALL accounts to maximize results
370
+ rows = conn.execute("SELECT session_string, api_id, api_hash, phone_number FROM accounts WHERE user_id = ? AND status = 'active'", (user_id,)).fetchall()
371
 
372
+ if not rows: raise HTTPException(status_code=404, detail="No active accounts found.")
 
 
373
 
374
+ unique_groups: Dict[str, Dict[str, Any]] = {}
375
+
376
+ # Iterate through accounts to get as many results as possible
377
+ # Telegram results vary by account/region/trust
378
+ for row in rows:
379
+ session_str, api_id, api_hash, acc_phone = row[0], int(row[1]), row[2], row[3]
380
+ client = TelegramClient(StringSession(session_str), api_id, api_hash)
381
+ try:
382
+ await client.connect()
383
+ if not await client.is_user_authorized():
384
+ continue
385
 
386
+ # Requesting high limit. Merge with what we already have.
387
+ search_result = await client(SearchRequest(q=query, limit=limit))
 
 
 
 
388
 
389
+ for chat in search_result.chats:
390
+ # We skip deleted chats but take everything else (even min objects)
391
+ if not chat: continue
392
+
393
+ chat_id_str = str(getattr(chat, 'id', '0'))
394
+ if chat_id_str in unique_groups: continue
395
+
396
+ is_private = getattr(chat, 'username', None) is None
397
+
398
+ title = getattr(chat, 'title', 'Unknown')
399
+ username = getattr(chat, 'username', None)
400
+
401
+ # Metadata extraction
402
+ participants_count = getattr(chat, 'participants_count', 0) or getattr(chat, 'megagroup_participants', 0) or 0
403
+ chat_type = 'channel' if getattr(chat, 'broadcast', False) else 'group'
404
+
405
+ # Check history
406
+ conn2 = get_db_connection()
407
+ # Isolation: Ensure we track this for the specific user
408
+ conn2.execute("INSERT OR IGNORE INTO scrape_history (group_id, user_id) VALUES (?, ?)", (chat_id_str, user_id))
409
+ if hasattr(conn2, "commit"): conn2.commit()
410
+ global_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ?", (chat_id_str,)).fetchone()[0]
411
+ user_count = conn2.execute("SELECT COUNT(*) FROM scrape_history WHERE group_id = ? AND user_id = ?", (chat_id_str, user_id)).fetchone()[0]
412
+ if hasattr(conn2, "close"): conn2.close()
413
+
414
+ unique_groups[chat_id_str] = {
415
+ "id": chat_id_str,
416
+ "title": title,
417
+ "username": username,
418
+ "participants_count": participants_count,
419
+ "type": chat_type,
420
+ "is_private": is_private,
421
+ "country": "Global",
422
+ "global_shows": global_count,
423
+ "user_shows": user_count
424
+ }
425
+ except Exception as e:
426
+ print(f"SCRAPER: Account {acc_phone} error: {e}")
427
+ finally:
428
+ await client.disconnect()
429
+
430
+ if hasattr(conn, "close"): conn.close()
431
+ return {"groups": list(unique_groups.values())}
432
 
433
  class BulkJoinRequest(BaseModel):
434
  group_ids: list[str]