Reooiq commited on
Commit
d837e7a
·
verified ·
1 Parent(s): 71cb29f

Update bot.py

Browse files
Files changed (1) hide show
  1. bot.py +164 -132
bot.py CHANGED
@@ -35,9 +35,9 @@ SESSION_STRING = os.environ.get("SESSION_STRING", "")
35
  TARGET_CHAT_RAW = os.environ.get("TARGET_CHAT", "me")
36
  AUTO_SCAN_ALL = os.environ.get("AUTO_SCAN_ALL", "1") == "1"
37
 
38
- DATA_DIR = "/data" if os.path.exists("/data") else "/tmp/data"
39
  DOWNLOAD_DIR = os.path.join(DATA_DIR, "downloads")
40
- DB_PATH = os.path.join(DATA_DIR, "archiver_final.db")
41
  os.makedirs(DOWNLOAD_DIR, exist_ok=True)
42
 
43
  app = Client(
@@ -50,7 +50,7 @@ app = Client(
50
  max_concurrent_transmissions=4,
51
  device_model="Archiver Pro",
52
  system_version="Linux 64bit",
53
- app_version="6.1.0",
54
  lang_code="ar"
55
  )
56
 
@@ -96,66 +96,121 @@ class SmartFloodManager:
96
 
97
  flood_manager = SmartFloodManager(normal_delay=1.5)
98
 
99
- def cleanup_corrupted_files():
100
- for ext in ["", "-wal", "-shm", "-journal"]:
101
- p = f"{DB_PATH}{ext}"
102
- if os.path.exists(p):
103
- try:
104
- os.remove(p)
105
- except Exception as e:
106
- logger.error(f"Failed removing {p}: {e}")
107
-
108
- async def init_db():
109
- if os.path.exists(DB_PATH):
110
- try:
111
- async with aiosqlite.connect(DB_PATH) as db:
112
- cur = await db.execute("PRAGMA integrity_check;")
113
- res = await cur.fetchone()
114
- if not res or res[0] != "ok":
115
- raise Exception("Integrity check failed")
116
- except Exception as e:
117
- logger.error(f"Database reset due to: {e}")
118
- cleanup_corrupted_files()
119
 
120
- async with aiosqlite.connect(DB_PATH) as db:
121
- await db.execute("PRAGMA journal_mode=WAL;")
122
- await db.execute("PRAGMA synchronous=NORMAL;")
123
- await db.execute("PRAGMA busy_timeout=30000;")
124
-
125
- await db.execute("""
126
- CREATE TABLE IF NOT EXISTS jobs (
127
- id INTEGER PRIMARY KEY AUTOINCREMENT,
128
- chat_id INTEGER NOT NULL,
129
- message_id INTEGER NOT NULL,
130
- media_group_id TEXT,
131
- media_type TEXT,
132
- caption TEXT,
133
- source_name TEXT,
134
- source_username TEXT,
135
- album_total INTEGER DEFAULT 1,
136
- album_index INTEGER DEFAULT 1,
137
- status TEXT DEFAULT 'pending',
138
- attempts INTEGER DEFAULT 0,
139
- error TEXT,
140
- created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
141
- UNIQUE(chat_id, message_id)
142
- )
143
- """)
144
- await db.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)")
145
- await db.execute("CREATE INDEX IF NOT EXISTS idx_jobs_media_group ON jobs(media_group_id)")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
146
 
147
- await db.execute("""
148
- CREATE TABLE IF NOT EXISTS scrape_checkpoints (
149
- chat_id INTEGER PRIMARY KEY,
150
- chat_title TEXT,
151
- chat_type TEXT,
152
- last_scraped_id INTEGER DEFAULT 0,
153
- total_queued INTEGER DEFAULT 0,
154
- status TEXT DEFAULT 'idle',
155
- updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
156
- )
157
- """)
158
- await db.commit()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
 
160
  async def add_job_to_queue(message: Message, album_info: Optional[Dict[str, int]] = None) -> bool:
161
  if not message.media:
@@ -186,18 +241,15 @@ async def add_job_to_queue(message: Message, album_info: Optional[Dict[str, int]
186
  album_index = album_info.get("index", 1)
187
 
188
  try:
189
- async with aiosqlite.connect(DB_PATH) as db:
190
- await db.execute("PRAGMA busy_timeout=30000;")
191
- cursor = await db.execute("""
192
- INSERT OR IGNORE INTO jobs
193
- (chat_id, message_id, media_group_id, media_type, caption, source_name, source_username, album_total, album_index, status)
194
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
195
- """, (chat_id, message_id, media_group_id, media_type, raw_caption, source_name, source_username, album_total, album_index))
196
- await db.commit()
197
- if cursor.rowcount > 0:
198
- stats["total_queued"] += 1
199
- return True
200
- return False
201
  except Exception as e:
202
  logger.error(f"Error queuing message {message_id}: {e}")
203
  return False
@@ -317,39 +369,28 @@ async def queue_worker_loop():
317
  await asyncio.sleep(5)
318
  continue
319
 
320
- job = None
321
- async with aiosqlite.connect(DB_PATH) as db:
322
- db.row_factory = aiosqlite.Row
323
- await db.execute("PRAGMA busy_timeout=30000;")
324
- cursor = await db.execute("""
325
- SELECT * FROM jobs
326
- WHERE status = 'pending' AND attempts < 10
327
- ORDER BY attempts ASC, id ASC LIMIT 1
328
- """)
329
- row = await cursor.fetchone()
330
- if not row:
331
- await asyncio.sleep(2)
332
- continue
333
-
334
- job = dict(row)
335
- await db.execute("UPDATE jobs SET status='processing' WHERE id=?", (job["id"],))
336
- await db.commit()
337
 
 
338
  result = await process_job(job)
339
 
340
- async with aiosqlite.connect(DB_PATH) as db:
341
- await db.execute("PRAGMA busy_timeout=30000;")
342
- if result == "SUCCESS":
343
- await db.execute("UPDATE jobs SET status='completed', error=NULL WHERE id=?", (job["id"],))
344
- stats["total_transferred"] += 1
345
- elif result == "DELETED":
346
- await db.execute("UPDATE jobs SET status='failed', error='Deleted' WHERE id=?", (job["id"],))
347
- stats["total_failed"] += 1
348
- else:
349
- await db.execute("""
350
- UPDATE jobs SET status='pending', attempts=attempts+1, error='Retry' WHERE id=?
351
- """, (job["id"],))
352
- await db.commit()
353
 
354
  except (FloodWait, FloodPremiumWait) as e:
355
  sec = getattr(e, 'value', 30)
@@ -395,13 +436,8 @@ async def warm_up_dialogs_cache() -> List[Dict[str, Any]]:
395
 
396
  async def scrape_single_chat(chat_id: int, chat_title: str, limit: int = 0) -> int:
397
  try:
398
- last_id = 0
399
- async with aiosqlite.connect(DB_PATH) as db:
400
- await db.execute("PRAGMA busy_timeout=30000;")
401
- cursor = await db.execute("SELECT last_scraped_id FROM scrape_checkpoints WHERE chat_id=?", (chat_id,))
402
- row = await cursor.fetchone()
403
- if row:
404
- last_id = row[0]
405
 
406
  logger.info(f"🔍 Scraping: {chat_title} (from msg {last_id})...")
407
  count = 0
@@ -424,16 +460,14 @@ async def scrape_single_chat(chat_id: int, chat_title: str, limit: int = 0) -> i
424
  count += 1
425
 
426
  if count > 0 and count % 100 == 0:
427
- async with aiosqlite.connect(DB_PATH) as db:
428
- await db.execute("""
429
- INSERT INTO scrape_checkpoints (chat_id, chat_title, last_scraped_id, total_queued, status)
430
- VALUES (?, ?, ?, ?, 'in_progress')
431
- ON CONFLICT(chat_id) DO UPDATE SET
432
- last_scraped_id=excluded.last_scraped_id,
433
- total_queued=total_queued+excluded.total_queued,
434
- updated_at=CURRENT_TIMESTAMP
435
- """, (chat_id, chat_title, highest_msg_id, 100))
436
- await db.commit()
437
  await asyncio.sleep(0.1)
438
 
439
  for mg_id, msgs in media_groups.items():
@@ -443,17 +477,15 @@ async def scrape_single_chat(chat_id: int, chat_title: str, limit: int = 0) -> i
443
  if await add_job_to_queue(msg, album_info={"total": total, "index": idx}):
444
  count += 1
445
 
446
- async with aiosqlite.connect(DB_PATH) as db:
447
- await db.execute("""
448
- INSERT INTO scrape_checkpoints (chat_id, chat_title, last_scraped_id, total_queued, status)
449
- VALUES (?, ?, ?, ?, 'completed')
450
- ON CONFLICT(chat_id) DO UPDATE SET
451
- last_scraped_id=excluded.last_scraped_id,
452
- total_queued=total_queued+excluded.total_queued,
453
- status='completed',
454
- updated_at=CURRENT_TIMESTAMP
455
- """, (chat_id, chat_title, highest_msg_id, count))
456
- await db.commit()
457
 
458
  logger.info(f"✅ Finished {chat_title}: queued {count} files.")
459
  return count
@@ -602,7 +634,7 @@ async def main():
602
  logger.info(f"Logged in: {me.first_name} (@{me.username or me.id})")
603
 
604
  await resolve_target_chat()
605
- await init_db()
606
 
607
  asyncio.create_task(queue_worker_loop())
608
 
 
35
  TARGET_CHAT_RAW = os.environ.get("TARGET_CHAT", "me")
36
  AUTO_SCAN_ALL = os.environ.get("AUTO_SCAN_ALL", "1") == "1"
37
 
38
+ DATA_DIR = "/tmp/data"
39
  DOWNLOAD_DIR = os.path.join(DATA_DIR, "downloads")
40
+ DB_PATH = os.path.join(DATA_DIR, "archiver_v7.db")
41
  os.makedirs(DOWNLOAD_DIR, exist_ok=True)
42
 
43
  app = Client(
 
50
  max_concurrent_transmissions=4,
51
  device_model="Archiver Pro",
52
  system_version="Linux 64bit",
53
+ app_version="7.0.0",
54
  lang_code="ar"
55
  )
56
 
 
96
 
97
  flood_manager = SmartFloodManager(normal_delay=1.5)
98
 
99
+ class SafeDBManager:
100
+ def __init__(self, db_path: str):
101
+ self.db_path = db_path
102
+ self.lock = asyncio.Lock()
103
+ self.conn: Optional[aiosqlite.Connection] = None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
+ async def connect(self):
106
+ async with self.lock:
107
+ if self.conn:
108
+ try:
109
+ await self.conn.close()
110
+ except Exception:
111
+ pass
112
+ self.conn = None
113
+
114
+ if os.path.exists(self.db_path):
115
+ try:
116
+ test_conn = await aiosqlite.connect(self.db_path)
117
+ cur = await test_conn.execute("PRAGMA integrity_check;")
118
+ res = await cur.fetchone()
119
+ await test_conn.close()
120
+ if not res or res[0] != "ok":
121
+ raise Exception("Corrupt DB check failed")
122
+ except Exception as e:
123
+ logger.error(f"Resetting corrupt DB: {e}")
124
+ for ext in ["", "-wal", "-shm", "-journal"]:
125
+ f = f"{self.db_path}{ext}"
126
+ if os.path.exists(f):
127
+ try:
128
+ os.remove(f)
129
+ except Exception:
130
+ pass
131
+
132
+ self.conn = await aiosqlite.connect(self.db_path)
133
+ self.conn.row_factory = aiosqlite.Row
134
+ await self.conn.execute("PRAGMA journal_mode=WAL;")
135
+ await self.conn.execute("PRAGMA synchronous=NORMAL;")
136
+ await self.conn.execute("PRAGMA busy_timeout=60000;")
137
+
138
+ await self.conn.execute("""
139
+ CREATE TABLE IF NOT EXISTS jobs (
140
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
141
+ chat_id INTEGER NOT NULL,
142
+ message_id INTEGER NOT NULL,
143
+ media_group_id TEXT,
144
+ media_type TEXT,
145
+ caption TEXT,
146
+ source_name TEXT,
147
+ source_username TEXT,
148
+ album_total INTEGER DEFAULT 1,
149
+ album_index INTEGER DEFAULT 1,
150
+ status TEXT DEFAULT 'pending',
151
+ attempts INTEGER DEFAULT 0,
152
+ error TEXT,
153
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
154
+ UNIQUE(chat_id, message_id)
155
+ )
156
+ """)
157
+ await self.conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)")
158
+ await self.conn.execute("CREATE INDEX IF NOT EXISTS idx_jobs_media_group ON jobs(media_group_id)")
159
+
160
+ await self.conn.execute("""
161
+ CREATE TABLE IF NOT EXISTS scrape_checkpoints (
162
+ chat_id INTEGER PRIMARY KEY,
163
+ chat_title TEXT,
164
+ chat_type TEXT,
165
+ last_scraped_id INTEGER DEFAULT 0,
166
+ total_queued INTEGER DEFAULT 0,
167
+ status TEXT DEFAULT 'idle',
168
+ updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
169
+ )
170
+ """)
171
+ await self.conn.commit()
172
+ logger.info("Database initialized with single connection lock.")
173
 
174
+ async def execute(self, query: str, params: tuple = ()) -> aiosqlite.Cursor:
175
+ async with self.lock:
176
+ try:
177
+ cur = await self.conn.execute(query, params)
178
+ await self.conn.commit()
179
+ return cur
180
+ except aiosqlite.DatabaseError as e:
181
+ logger.error(f"Database error during query, auto-recovering: {e}")
182
+ await self.connect()
183
+ cur = await self.conn.execute(query, params)
184
+ await self.conn.commit()
185
+ return cur
186
+
187
+ async def fetchone(self, query: str, params: tuple = ()):
188
+ async with self.lock:
189
+ try:
190
+ cur = await self.conn.execute(query, params)
191
+ row = await cur.fetchone()
192
+ return dict(row) if row else None
193
+ except aiosqlite.DatabaseError as e:
194
+ logger.error(f"Database error during fetchone, auto-recovering: {e}")
195
+ await self.connect()
196
+ cur = await self.conn.execute(query, params)
197
+ row = await cur.fetchone()
198
+ return dict(row) if row else None
199
+
200
+ async def fetchall(self, query: str, params: tuple = ()):
201
+ async with self.lock:
202
+ try:
203
+ cur = await self.conn.execute(query, params)
204
+ rows = await cur.fetchall()
205
+ return [dict(r) for r in rows]
206
+ except aiosqlite.DatabaseError as e:
207
+ logger.error(f"Database error during fetchall, auto-recovering: {e}")
208
+ await self.connect()
209
+ cur = await self.conn.execute(query, params)
210
+ rows = await cur.fetchall()
211
+ return [dict(r) for r in rows]
212
+
213
+ db = SafeDBManager(DB_PATH)
214
 
215
  async def add_job_to_queue(message: Message, album_info: Optional[Dict[str, int]] = None) -> bool:
216
  if not message.media:
 
241
  album_index = album_info.get("index", 1)
242
 
243
  try:
244
+ cur = await db.execute("""
245
+ INSERT OR IGNORE INTO jobs
246
+ (chat_id, message_id, media_group_id, media_type, caption, source_name, source_username, album_total, album_index, status)
247
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending')
248
+ """, (chat_id, message_id, media_group_id, media_type, raw_caption, source_name, source_username, album_total, album_index))
249
+ if cur.rowcount > 0:
250
+ stats["total_queued"] += 1
251
+ return True
252
+ return False
 
 
 
253
  except Exception as e:
254
  logger.error(f"Error queuing message {message_id}: {e}")
255
  return False
 
369
  await asyncio.sleep(5)
370
  continue
371
 
372
+ job = await db.fetchone("""
373
+ SELECT * FROM jobs
374
+ WHERE status = 'pending' AND attempts < 10
375
+ ORDER BY attempts ASC, id ASC LIMIT 1
376
+ """)
377
+ if not job:
378
+ await asyncio.sleep(2)
379
+ continue
 
 
 
 
 
 
 
 
 
380
 
381
+ await db.execute("UPDATE jobs SET status='processing' WHERE id=?", (job["id"],))
382
  result = await process_job(job)
383
 
384
+ if result == "SUCCESS":
385
+ await db.execute("UPDATE jobs SET status='completed', error=NULL WHERE id=?", (job["id"],))
386
+ stats["total_transferred"] += 1
387
+ elif result == "DELETED":
388
+ await db.execute("UPDATE jobs SET status='failed', error='Deleted' WHERE id=?", (job["id"],))
389
+ stats["total_failed"] += 1
390
+ else:
391
+ await db.execute("""
392
+ UPDATE jobs SET status='pending', attempts=attempts+1, error='Retry' WHERE id=?
393
+ """, (job["id"],))
 
 
 
394
 
395
  except (FloodWait, FloodPremiumWait) as e:
396
  sec = getattr(e, 'value', 30)
 
436
 
437
  async def scrape_single_chat(chat_id: int, chat_title: str, limit: int = 0) -> int:
438
  try:
439
+ last_row = await db.fetchone("SELECT last_scraped_id FROM scrape_checkpoints WHERE chat_id=?", (chat_id,))
440
+ last_id = last_row["last_scraped_id"] if last_row else 0
 
 
 
 
 
441
 
442
  logger.info(f"🔍 Scraping: {chat_title} (from msg {last_id})...")
443
  count = 0
 
460
  count += 1
461
 
462
  if count > 0 and count % 100 == 0:
463
+ await db.execute("""
464
+ INSERT INTO scrape_checkpoints (chat_id, chat_title, last_scraped_id, total_queued, status)
465
+ VALUES (?, ?, ?, ?, 'in_progress')
466
+ ON CONFLICT(chat_id) DO UPDATE SET
467
+ last_scraped_id=excluded.last_scraped_id,
468
+ total_queued=total_queued+excluded.total_queued,
469
+ updated_at=CURRENT_TIMESTAMP
470
+ """, (chat_id, chat_title, highest_msg_id, 100))
 
 
471
  await asyncio.sleep(0.1)
472
 
473
  for mg_id, msgs in media_groups.items():
 
477
  if await add_job_to_queue(msg, album_info={"total": total, "index": idx}):
478
  count += 1
479
 
480
+ await db.execute("""
481
+ INSERT INTO scrape_checkpoints (chat_id, chat_title, last_scraped_id, total_queued, status)
482
+ VALUES (?, ?, ?, ?, 'completed')
483
+ ON CONFLICT(chat_id) DO UPDATE SET
484
+ last_scraped_id=excluded.last_scraped_id,
485
+ total_queued=total_queued+excluded.total_queued,
486
+ status='completed',
487
+ updated_at=CURRENT_TIMESTAMP
488
+ """, (chat_id, chat_title, highest_msg_id, count))
 
 
489
 
490
  logger.info(f"✅ Finished {chat_title}: queued {count} files.")
491
  return count
 
634
  logger.info(f"Logged in: {me.first_name} (@{me.username or me.id})")
635
 
636
  await resolve_target_chat()
637
+ await db.connect()
638
 
639
  asyncio.create_task(queue_worker_loop())
640