Reooiq commited on
Commit
e3d3cdf
ยท
verified ยท
1 Parent(s): e6b798c

Update bot.py

Browse files
Files changed (1) hide show
  1. bot.py +50 -43
bot.py CHANGED
@@ -5,21 +5,15 @@ import time
5
  import threading
6
  import aiosqlite
7
  import gradio as gr
8
- from collections import defaultdict
9
- from pyrogram import Client, filters
10
- from pyrogram.errors import FloodWait
11
- from pyrogram.types import (
12
  InputMediaPhoto, InputMediaVideo, InputMediaDocument,
13
  InputMediaAudio
14
  )
15
  from fastapi import FastAPI
16
  from tenacity import retry, stop_after_attempt, wait_exponential_jitter, retry_if_exception_type
17
 
18
- # ============ ุฅุตู„ุงุญ MIN_CHANNEL_ID ============
19
- import pyrogram
20
- pyrogram.utils.MIN_CHANNEL_ID = -1007852516352
21
- pyrogram.utils.MIN_CHAT_ID = -999999999999
22
-
23
  logging.basicConfig(
24
  level=logging.INFO,
25
  format='%(asctime)s - %(levelname)s - %(message)s'
@@ -37,14 +31,14 @@ DOWNLOAD_DIR = os.path.join(DATA_DIR, "downloads")
37
  DB_PATH = os.path.join(DATA_DIR, "queue.db")
38
  os.makedirs(DOWNLOAD_DIR, exist_ok=True)
39
 
40
- # ============ ุงู„ุนู…ูŠู„ ============
41
  app = Client(
42
  "archiver",
43
  api_id=API_ID,
44
  api_hash=API_HASH,
45
  session_string=SESSION_STRING,
46
  in_memory=True,
47
- workers=1,
48
  device_model="Samsung Galaxy S23 Ultra",
49
  system_version="Android 14",
50
  app_version="10.14.1",
@@ -55,24 +49,26 @@ TARGET_CHAT = None
55
 
56
  # ============ Throttle Controller ============
57
  class ThrottleController:
58
- def __init__(self, rate=1.2):
59
  self.rate = rate
60
- self.last_send = defaultdict(float)
61
  self.lock = asyncio.Lock()
62
 
63
  async def wait(self, chat_id):
64
  async with self.lock:
65
  now = time.time()
66
- elapsed = now - self.last_send[chat_id]
 
67
  if elapsed < self.rate:
68
- wait_time = self.rate - elapsed
69
- await asyncio.sleep(wait_time)
70
  self.last_send[chat_id] = time.time()
71
 
72
- throttle = ThrottleController(rate=1.2)
73
 
74
  # ============ Semaphores ============
75
- download_sem = asyncio.Semaphore(2)
 
 
76
  send_sem = asyncio.Semaphore(2)
77
 
78
  # ============ ู‚ุงุนุฏุฉ ุงู„ุจูŠุงู†ุงุช ============
@@ -99,12 +95,8 @@ async def init_db():
99
  UNIQUE(chat_id, message_id)
100
  )
101
  """)
102
- await db.execute("""
103
- CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)
104
- """)
105
- await db.execute("""
106
- CREATE INDEX IF NOT EXISTS idx_jobs_media_group ON jobs(media_group_id)
107
- """)
108
  await db.commit()
109
  logger.info("Database ready")
110
 
@@ -153,7 +145,7 @@ async def add_job(message):
153
  (chat_id, message_id, media_group_id, media_type, file_id, file_name, file_size, caption, source_name, status)
154
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
155
  """, (
156
- message.chat.id, message.id, message.media_group_id,
157
  m_type, file_id, file_name, file_size, final_cap, src_name
158
  ))
159
  await db.commit()
@@ -179,7 +171,8 @@ async def update_job_status(chat_id, message_id, status, file_path=None, error=N
179
  """, (status, chat_id, message_id))
180
  await db.commit()
181
 
182
- async def get_pending_jobs(limit=3):
 
183
  async with aiosqlite.connect(DB_PATH) as db:
184
  db.row_factory = aiosqlite.Row
185
  cursor = await db.execute("""
@@ -253,7 +246,7 @@ def get_source_name(message):
253
  return chat.first_name or chat.username or "Private"
254
  return "Source"
255
 
256
- # ============ ุงู„ุชุญู…ูŠู„ ู…ุน ุงู„ุชุญู‚ู‚ ============
257
  async def verify_download(file_path, expected_size):
258
  if not file_path or not os.path.exists(file_path):
259
  return False
@@ -267,6 +260,7 @@ async def verify_download(file_path, expected_size):
267
  return True
268
 
269
  async def download_job(job):
 
270
  chat_id = job['chat_id']
271
  message_id = job['message_id']
272
  expected_size = job['file_size'] or 0
@@ -282,6 +276,7 @@ async def download_job(job):
282
  try:
283
  await update_job_status(chat_id, message_id, 'downloading')
284
 
 
285
  downloaded = await app.download_media(
286
  job['file_id'],
287
  file_name=file_path
@@ -290,6 +285,7 @@ async def download_job(job):
290
  if not downloaded:
291
  raise Exception("download_media returned None")
292
 
 
293
  if not await verify_download(downloaded, expected_size):
294
  logger.info(f"Retrying download {message_id}")
295
  if os.path.exists(downloaded):
@@ -305,6 +301,11 @@ async def download_job(job):
305
  logger.info(f"Download complete: {message_id} | {os.path.getsize(downloaded)} bytes")
306
  return True
307
 
 
 
 
 
 
308
  except Exception as e:
309
  await update_job_status(chat_id, message_id, 'pending', error=str(e))
310
  logger.error(f"Download failed {message_id}: {e}")
@@ -423,29 +424,33 @@ async def send_album_group(media_group_id):
423
 
424
  # ============ Workers ============
425
  async def download_worker():
426
- logger.info("Download Worker started")
427
- await asyncio.sleep(3)
 
428
 
429
  while True:
430
  try:
431
  if not app.is_connected:
432
- logger.warning("Pyrogram not connected, waiting...")
433
  await asyncio.sleep(5)
434
  continue
435
 
436
- jobs = await get_pending_jobs(limit=3)
437
  if not jobs:
438
- await asyncio.sleep(2)
439
  continue
440
 
441
- tasks = []
442
- for job in jobs:
443
- if len(tasks) >= 2:
444
- break
445
- tasks.append(download_job(job))
446
 
447
- if tasks:
448
- await asyncio.gather(*tasks, return_exceptions=True)
 
 
 
 
 
 
 
449
 
450
  except Exception as e:
451
  logger.error(f"Download Worker error: {e}")
@@ -453,12 +458,12 @@ async def download_worker():
453
 
454
  async def send_worker():
455
  logger.info("Send Worker started")
456
- await asyncio.sleep(5)
457
 
458
  while True:
459
  try:
460
  if not app.is_connected:
461
- logger.warning("Pyrogram not connected, waiting...")
462
  await asyncio.sleep(5)
463
  continue
464
 
@@ -485,11 +490,13 @@ async def send_worker():
485
  break
486
  async with send_sem:
487
  await send_job(job)
 
488
 
489
  for (mg_id,) in album_rows:
490
  if not app.is_connected:
491
  break
492
  await send_album_group(mg_id)
 
493
 
494
  await asyncio.sleep(1)
495
 
@@ -552,8 +559,8 @@ async def resolve_target_chat():
552
  def run_gradio():
553
  with gr.Blocks(title="Telegram Archiver Pro") as demo:
554
  gr.Markdown("# Telegram Archiver Pro")
555
- gr.Markdown("Status: Running")
556
- gr.Textbox(label="Status", value="Active | SQLite Queue | 3 Workers", interactive=False)
557
 
558
  fastapi_app = FastAPI()
559
 
 
5
  import threading
6
  import aiosqlite
7
  import gradio as gr
8
+ from hydrogram import Client, filters
9
+ from hydrogram.errors import FloodWait
10
+ from hydrogram.types import (
 
11
  InputMediaPhoto, InputMediaVideo, InputMediaDocument,
12
  InputMediaAudio
13
  )
14
  from fastapi import FastAPI
15
  from tenacity import retry, stop_after_attempt, wait_exponential_jitter, retry_if_exception_type
16
 
 
 
 
 
 
17
  logging.basicConfig(
18
  level=logging.INFO,
19
  format='%(asctime)s - %(levelname)s - %(message)s'
 
31
  DB_PATH = os.path.join(DATA_DIR, "queue.db")
32
  os.makedirs(DOWNLOAD_DIR, exist_ok=True)
33
 
34
+ # ============ ุงู„ุนู…ูŠู„ (Hydrogram) ============
35
  app = Client(
36
  "archiver",
37
  api_id=API_ID,
38
  api_hash=API_HASH,
39
  session_string=SESSION_STRING,
40
  in_memory=True,
41
+ workers=4, # Hydrogram ูŠุชุนุงู…ู„ ุฃูุถู„ ู…ุน workers ุฃุนู„ู‰
42
  device_model="Samsung Galaxy S23 Ultra",
43
  system_version="Android 14",
44
  app_version="10.14.1",
 
49
 
50
  # ============ Throttle Controller ============
51
  class ThrottleController:
52
+ def __init__(self, rate=2.0):
53
  self.rate = rate
54
+ self.last_send = {}
55
  self.lock = asyncio.Lock()
56
 
57
  async def wait(self, chat_id):
58
  async with self.lock:
59
  now = time.time()
60
+ last = self.last_send.get(chat_id, 0)
61
+ elapsed = now - last
62
  if elapsed < self.rate:
63
+ await asyncio.sleep(self.rate - elapsed)
 
64
  self.last_send[chat_id] = time.time()
65
 
66
+ throttle = ThrottleController(rate=2.0)
67
 
68
  # ============ Semaphores ============
69
+ # ๐Ÿ”‘ Serial downloading: ู…ู„ู ูˆุงุญุฏ ูู‚ุท ูŠูุญู…ู„ ููŠ ูƒู„ ูˆู‚ุช (ู„ู…ู†ุน Media DC storm)
70
+ download_lock = asyncio.Lock()
71
+ # ุฅุฑุณุงู„: 2 ู…ุชูˆุงุฒูŠ ูƒุญุฏ ุฃู‚ุตู‰
72
  send_sem = asyncio.Semaphore(2)
73
 
74
  # ============ ู‚ุงุนุฏุฉ ุงู„ุจูŠุงู†ุงุช ============
 
95
  UNIQUE(chat_id, message_id)
96
  )
97
  """)
98
+ await db.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)")
99
+ await db.execute("CREATE INDEX IF NOT EXISTS idx_jobs_media_group ON jobs(media_group_id)")
 
 
 
 
100
  await db.commit()
101
  logger.info("Database ready")
102
 
 
145
  (chat_id, message_id, media_group_id, media_type, file_id, file_name, file_size, caption, source_name, status)
146
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
147
  """, (
148
+ message.chat.id, message.id, str(message.media_group_id) if message.media_group_id else None,
149
  m_type, file_id, file_name, file_size, final_cap, src_name
150
  ))
151
  await db.commit()
 
171
  """, (status, chat_id, message_id))
172
  await db.commit()
173
 
174
+ async def get_pending_jobs(limit=1):
175
+ """๐Ÿ”‘ ุฌู„ุจ ู…ู‡ู…ุฉ ูˆุงุญุฏุฉ ูู‚ุท (Serial downloading)"""
176
  async with aiosqlite.connect(DB_PATH) as db:
177
  db.row_factory = aiosqlite.Row
178
  cursor = await db.execute("""
 
246
  return chat.first_name or chat.username or "Private"
247
  return "Source"
248
 
249
+ # ============ ุงู„ุชุญู…ูŠู„ ุงู„ู…ุชุณู„ุณู„ (Serial) ============
250
  async def verify_download(file_path, expected_size):
251
  if not file_path or not os.path.exists(file_path):
252
  return False
 
260
  return True
261
 
262
  async def download_job(job):
263
+ """๐Ÿ”‘ ุชุญู…ูŠู„ ู…ุชุณู„ุณู„: ู…ู„ู ูˆุงุญุฏ ููŠ ูƒู„ ูˆู‚ุช"""
264
  chat_id = job['chat_id']
265
  message_id = job['message_id']
266
  expected_size = job['file_size'] or 0
 
276
  try:
277
  await update_job_status(chat_id, message_id, 'downloading')
278
 
279
+ # ๐Ÿ”‘ ุงู„ุชุญู…ูŠู„ ุจุงุณุชุฎุฏุงู… Hydrogram
280
  downloaded = await app.download_media(
281
  job['file_id'],
282
  file_name=file_path
 
285
  if not downloaded:
286
  raise Exception("download_media returned None")
287
 
288
+ # ุงู„ุชุญู‚ู‚ ู…ู† ุงู„ุงูƒุชู…ุงู„
289
  if not await verify_download(downloaded, expected_size):
290
  logger.info(f"Retrying download {message_id}")
291
  if os.path.exists(downloaded):
 
301
  logger.info(f"Download complete: {message_id} | {os.path.getsize(downloaded)} bytes")
302
  return True
303
 
304
+ except FloodWait as e:
305
+ logger.warning(f"FloodWait during download: {e.value}s")
306
+ await update_job_status(chat_id, message_id, 'pending', error=f"FloodWait:{e.value}")
307
+ await asyncio.sleep(e.value)
308
+ return False
309
  except Exception as e:
310
  await update_job_status(chat_id, message_id, 'pending', error=str(e))
311
  logger.error(f"Download failed {message_id}: {e}")
 
424
 
425
  # ============ Workers ============
426
  async def download_worker():
427
+ """๐Ÿ”‘ Serial Download Worker: ู…ู„ู ูˆุงุญุฏ ููŠ ูƒู„ ูˆู‚ุช"""
428
+ logger.info("Serial Download Worker started")
429
+ await asyncio.sleep(5)
430
 
431
  while True:
432
  try:
433
  if not app.is_connected:
434
+ logger.warning("Hydrogram not connected, waiting...")
435
  await asyncio.sleep(5)
436
  continue
437
 
438
+ jobs = await get_pending_jobs(limit=1)
439
  if not jobs:
440
+ await asyncio.sleep(1)
441
  continue
442
 
443
+ job = jobs[0]
 
 
 
 
444
 
445
+ # ๐Ÿ”‘ Serial lock: ู…ู„ู ูˆุงุญุฏ ูู‚ุท ูŠูุญู…ู„
446
+ async with download_lock:
447
+ success = await download_job(job)
448
+
449
+ # ๐Ÿ”‘ ุชุฃุฎูŠุฑ 3 ุซูˆุงู†ูŠ ุจูŠู† ูƒู„ ุชุญู…ูŠู„ (ู„ู…ู†ุน FloodWait)
450
+ if success:
451
+ await asyncio.sleep(3)
452
+ else:
453
+ await asyncio.sleep(5) # ุชุฃุฎูŠุฑ ุฃุทูˆู„ ุฅุฐุง ูุดู„
454
 
455
  except Exception as e:
456
  logger.error(f"Download Worker error: {e}")
 
458
 
459
  async def send_worker():
460
  logger.info("Send Worker started")
461
+ await asyncio.sleep(7)
462
 
463
  while True:
464
  try:
465
  if not app.is_connected:
466
+ logger.warning("Hydrogram not connected, waiting...")
467
  await asyncio.sleep(5)
468
  continue
469
 
 
490
  break
491
  async with send_sem:
492
  await send_job(job)
493
+ await asyncio.sleep(2) # ุชุฃุฎูŠุฑ ุจูŠู† ุงู„ุฅุฑุณุงู„ูŠุงุช
494
 
495
  for (mg_id,) in album_rows:
496
  if not app.is_connected:
497
  break
498
  await send_album_group(mg_id)
499
+ await asyncio.sleep(2)
500
 
501
  await asyncio.sleep(1)
502
 
 
559
  def run_gradio():
560
  with gr.Blocks(title="Telegram Archiver Pro") as demo:
561
  gr.Markdown("# Telegram Archiver Pro")
562
+ gr.Markdown("Status: Running | Hydrogram | Serial Download")
563
+ gr.Textbox(label="Status", value="Active | SQLite Queue | Serial DL", interactive=False)
564
 
565
  fastapi_app = FastAPI()
566