saske560 commited on
Commit
dc2e8fc
·
verified ·
1 Parent(s): 85cca7a

Upload main.py

Browse files
Files changed (1) hide show
  1. main.py +21 -230
main.py CHANGED
@@ -158,10 +158,7 @@ VIDEO_EXTENSIONS = {
158
  PROGRESS_EDIT_EVERY = 5
159
  MAX_CPU_PERCENT = max(50, min(95, int(os.environ.get("MAX_CPU_PERCENT", "85"))))
160
  CPU_MONITOR_INTERVAL = max(0.5, float(os.environ.get("CPU_MONITOR_INTERVAL", "2.0")))
161
- COMPRESS_STALL_SECONDS = max(120, int(os.environ.get("COMPRESS_STALL_SECONDS", "420")))
162
- COMPRESS_START_GRACE_SECONDS = max(60, int(os.environ.get("COMPRESS_START_GRACE_SECONDS", "120")))
163
- FFMPEG_OUTPUT_IDLE_SECONDS = max(45, int(os.environ.get("FFMPEG_OUTPUT_IDLE_SECONDS", "120")))
164
- STALL_CPU_FLOOR_PERCENT = max(0.5, float(os.environ.get("STALL_CPU_FLOOR_PERCENT", "2.0")))
165
  QUEUE_WATCHDOG_INTERVAL = max(2.0, float(os.environ.get("QUEUE_WATCHDOG_INTERVAL", "5.0")))
166
  CPU_LIMIT_HEADROOM_PERCENT = max(1, min(15, int(os.environ.get("CPU_LIMIT_HEADROOM_PERCENT", "4"))))
167
 
@@ -983,9 +980,6 @@ class Job:
983
  cpu_usage_pct: float = 0.0
984
  cpu_slots: int = 0
985
  cpu_slots_max: int = 0
986
- ffmpeg_last_progress_at: float = 0.0
987
- ffmpeg_last_output_at: float = 0.0
988
- ffmpeg_last_stderr: str = ""
989
 
990
  ACTIVE_JOBS: Dict[str, Job] = {}
991
 
@@ -1128,43 +1122,6 @@ def _is_job_processing(job_id: str) -> bool:
1128
  return True
1129
  return any(uq.get("processing") == job_id for uq in _user_queues.values())
1130
 
1131
-
1132
- def _job_queue_position(job_id: str) -> Optional[tuple[str, int, int]]:
1133
- if _global_currently_processing == job_id:
1134
- return ("free", 0, max(1, len(_global_queue_list) + 1))
1135
- for i, item in enumerate(_global_queue_list, 1):
1136
- if item[3] == job_id:
1137
- return ("free", i, len(_global_queue_list))
1138
- for queue_key, uq in _user_queues.items():
1139
- if uq.get("processing") == job_id:
1140
- total = len(uq.get("list", [])) + 1
1141
- return (queue_key, 0, max(1, total))
1142
- for i, item in enumerate(uq.get("list", []), 1):
1143
- if item[3] == job_id:
1144
- total = len(uq.get("list", [])) + (1 if uq.get("processing") else 0)
1145
- return (queue_key, i, max(1, total))
1146
- return None
1147
-
1148
-
1149
- async def _edit_job_status(job: Job, text: str, *, remove_markup: bool = True):
1150
- try:
1151
- await bot.edit_message_text(
1152
- chat_id=job.chat_id,
1153
- message_id=job.status_msg_id,
1154
- text=text,
1155
- reply_markup=None if remove_markup else _cancel_markup(job.job_id),
1156
- )
1157
- except Exception:
1158
- pass
1159
-
1160
-
1161
- async def _force_finish_job(job: Job, reason: str, *, notify_user: bool = True):
1162
- job.cancelled = True
1163
- job.is_processing = False
1164
- if notify_user:
1165
- await _edit_job_status(job, reason, remove_markup=True)
1166
- await _cleanup_job(job)
1167
-
1168
  # =====================
1169
  # UTILS
1170
  # =====================
@@ -1316,33 +1273,18 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1316
  job.cpu_slots = current_cpu_slots
1317
  job.cpu_slots_max = max_parallel_slots
1318
  job.cpu_usage_pct = 0.0
1319
- job.ffmpeg_last_progress_at = time.time()
1320
- job.ffmpeg_last_output_at = time.time()
1321
- job.ffmpeg_last_stderr = ""
1322
  _job_touch(job, stage="comprimiendo", progress=0)
1323
 
1324
  out_time_ms = 0
1325
- now0 = time.time()
1326
- last_progress_at = now0
1327
- last_output_at = now0
1328
- last_stderr_at = now0
1329
  stderr_tail = deque(maxlen=80)
1330
- t0 = now0
1331
  ffmpeg_cpu_pct_total = 0.0
1332
  ffmpeg_cpu_pct_quota = 0.0
1333
- proc_cpu_raw = 0.0
1334
- input_size_mb = 0.0
1335
- with contextlib.suppress(Exception):
1336
- input_size_mb = max(0.0, os.path.getsize(in_f) / (1024 * 1024))
1337
- dynamic_start_grace = max(COMPRESS_START_GRACE_SECONDS, min(300, int(input_size_mb * 0.35) + int(dur * 0.02 if dur else 0)))
1338
- dynamic_stall_limit = max(COMPRESS_STALL_SECONDS, min(1200, int(dur * 0.12) if dur else 0))
1339
 
1340
  async def _read_stdout():
1341
- nonlocal out_time_ms, last_progress_at, last_output_at
1342
  async for line in proc.stdout:
1343
- last_output_at = time.time()
1344
- if job:
1345
- job.ffmpeg_last_output_at = last_output_at
1346
  s = line.decode("utf-8", errors="replace").strip()
1347
  if "=" not in s:
1348
  continue
@@ -1353,32 +1295,21 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1353
  if new_out_time_ms >= out_time_ms:
1354
  out_time_ms = new_out_time_ms
1355
  last_progress_at = time.time()
1356
- if job:
1357
- job.ffmpeg_last_progress_at = last_progress_at
1358
  except Exception:
1359
  pass
1360
  elif k == "progress" and v == "end":
1361
  last_progress_at = time.time()
1362
- if job:
1363
- job.ffmpeg_last_progress_at = last_progress_at
1364
  break
1365
 
1366
  async def _read_stderr():
1367
- nonlocal last_output_at, last_stderr_at
1368
  while True:
1369
  line = await proc.stderr.readline()
1370
  if not line:
1371
  break
1372
- last_output_at = time.time()
1373
- last_stderr_at = last_output_at
1374
- decoded = line.decode("utf-8", errors="replace").strip()
1375
- stderr_tail.append(decoded)
1376
- if job:
1377
- job.ffmpeg_last_output_at = last_output_at
1378
- job.ffmpeg_last_stderr = decoded[:300]
1379
 
1380
  async def _cpu_guard():
1381
- nonlocal ffmpeg_cpu_pct_total, ffmpeg_cpu_pct_quota, current_cpu_slots, cpu_subset, active_jobs, proc_cpu_raw
1382
  if proc.pid is None:
1383
  return
1384
  try:
@@ -1419,7 +1350,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1419
  job.cpu_usage_pct = ffmpeg_cpu_pct_total
1420
  job.cpu_slots = current_cpu_slots
1421
  job.cpu_slots_max = max_parallel_slots
1422
- job.ffmpeg_last_output_at = max(job.ffmpeg_last_output_at or 0.0, last_output_at)
1423
  _job_touch(job, stage="comprimiendo", progress=(job.progress_pct if job else None))
1424
  except Exception:
1425
  pass
@@ -1434,18 +1364,8 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1434
  while not wt.done():
1435
  if job and job.cancelled:
1436
  raise asyncio.CancelledError()
1437
- now = time.time()
1438
- elapsed = now - t0
1439
- progress_idle = now - last_progress_at
1440
- output_idle = now - max(last_output_at, last_stderr_at, last_progress_at)
1441
- no_progress_yet = out_time_ms <= 0
1442
- cpu_looks_idle = proc_cpu_raw < STALL_CPU_FLOOR_PERCENT
1443
- if no_progress_yet:
1444
- if elapsed > dynamic_start_grace and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and cpu_looks_idle:
1445
- raise RuntimeError("FFmpeg no arrancó bien la compresión y quedó sin actividad.")
1446
- else:
1447
- if progress_idle > dynamic_stall_limit and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and cpu_looks_idle:
1448
- raise RuntimeError("La compresión quedó atascada sin avance real durante demasiado tiempo.")
1449
  await asyncio.sleep(PROGRESS_EDIT_EVERY)
1450
  ds = out_time_ms / 1_000_000 if out_time_ms else 0.0
1451
  p = (ds / dur) if dur > 0 else 0.0
@@ -1466,7 +1386,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1466
  f"🧠 CPU: `{cpu_slots_txt} cores` | FFmpeg: `{ffmpeg_cpu_pct_total:.1f}%` host / `{ffmpeg_cpu_pct_quota:.1f}%` cuota\n"
1467
  f"🛡️ Tope(host): `{MAX_CPU_PERCENT}%` | Pool: `{max_parallel_slots}` | Jobs: `{active_jobs}`\n"
1468
  f"🧩 Cuota cgroup: `{quota_txt}` CPU | Capacidad: `{cpu_capacity:.2f}`\n"
1469
- f"🧯 Stall: `{int(progress_idle)}s/{dynamic_stall_limit}s` | Salida: `{int(output_idle)}s/{FFMPEG_OUTPUT_IDLE_SECONDS}s`\n"
1470
  f"⏳ ETA: `{_fmt_eta(eta)}`"
1471
  )
1472
  await wt
@@ -1489,10 +1408,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
1489
 
1490
  if proc.returncode != 0:
1491
  err = "\n".join([x for x in stderr_tail if x][-8:]) or "FFmpeg terminó con error."
1492
- if proc.returncode in (-9, 137):
1493
- err = "El proceso de compresión fue terminado por el sistema (memoria/recursos insuficientes)."
1494
- elif proc.returncode in (-15, 143) and not (job and job.cancelled):
1495
- err = "La compresión fue detenida antes de terminar. Revisa watchdog, reinicios del hosting o falta de recursos."
1496
  raise RuntimeError(err)
1497
  if not os.path.exists(out_f) or os.path.getsize(out_f) == 0:
1498
  raise RuntimeError("No se generó el archivo comprimido.")
@@ -1630,8 +1545,6 @@ async def _user_worker(queue_key: str):
1630
  # PROCESO PRINCIPAL
1631
  # =====================
1632
  async def _process(client, message: Message, status_msg: Message = None, job: Job = None):
1633
- if job:
1634
- job.task = asyncio.current_task()
1635
  editor = _Editor(status_msg, job_id=(job.job_id if job else None))
1636
  uid = message.from_user.id
1637
  cid = message.chat.id
@@ -1793,7 +1706,6 @@ async def _process(client, message: Message, status_msg: Message = None, job: Jo
1793
  pass
1794
  finally:
1795
  if job:
1796
- job.task = None
1797
  await _cleanup_job(job)
1798
  else:
1799
  for p in (in_f, out_f, thumb):
@@ -3678,16 +3590,11 @@ async def cmd_trabajos(client, msg: Message):
3678
 
3679
  lines = ["🧰 **Trabajos del bot**", "━━━━━━━━━━━━━━━━━━━━━"]
3680
  for idx, job in enumerate(sorted(active, key=lambda j: (j.is_processing, j.started_at or 0), reverse=True)[:20], 1):
3681
- qpos = _job_queue_position(job.job_id)
3682
- really_processing = bool(job.is_processing and (job.process and job.process.returncode is None or qpos and qpos[1] == 0 or job.task))
3683
- state = "🟢 Activo" if really_processing else "🟡 En espera"
3684
- queue_label = _plan_queue_label((qpos[0] if qpos else job.queue_name) or "free") if (qpos or job.queue_name) else "🧵 Cola pendiente"
3685
- extra_queue = ""
3686
- if qpos and qpos[1] > 0:
3687
- extra_queue = f" | Pos: `#{qpos[1]}/{qpos[2]}`"
3688
  lines.append(
3689
  f"{idx}. {state} | `{job.user_id}` | `{job.fname or job.job_id}`\n"
3690
- f" • Etapa: `{job.stage or 'pendiente'}` | Cola: `{queue_label}`{extra_queue}\n"
3691
  f" • Progreso: `{job.progress_pct:.1f}%` | CPU: `{job.cpu_usage_pct:.1f}%` | Slots: `{job.cpu_slots}/{job.cpu_slots_max}`\n"
3692
  f" • Última actividad: `{_fmt_eta(max(0, time.time() - (job.last_activity_at or time.time())))}` atrás | Tiempo: `{_job_elapsed(job)}`"
3693
  )
@@ -4272,7 +4179,6 @@ async def _queue_watchdog():
4272
  if _global_worker_task and _global_worker_task.done():
4273
  logger.warning("Watchdog: reiniciando worker global con %d elemento(s) pendientes.", len(_global_queue_list))
4274
  await _ensure_global_worker()
4275
- _global_queue_event.set()
4276
 
4277
  for queue_key, uq in list(_user_queues.items()):
4278
  processing_id = uq.get("processing")
@@ -4284,134 +4190,23 @@ async def _queue_watchdog():
4284
  if worker and worker.done():
4285
  logger.warning("Watchdog: reiniciando worker %s con %d elemento(s) pendientes.", queue_key, len(uq["list"]))
4286
  await _ensure_user_worker(queue_key)
4287
- uq["event"].set()
4288
 
4289
  now = time.time()
4290
- stale_limit = COMPRESS_STALL_SECONDS + 30
4291
  for job in list(ACTIVE_JOBS.values()):
4292
- if job.cancelled:
4293
- await _force_finish_job(job, "❌ **Trabajo cancelado por watchdog.**")
4294
  continue
4295
-
4296
- idle_for = now - (job.last_activity_at or now)
4297
- queue_pos = _job_queue_position(job.job_id)
4298
- task_done = bool(job.task and job.task.done())
4299
- proc_running = bool(job.process and job.process.returncode is None)
4300
-
4301
- if queue_pos is None and not proc_running and not job.is_processing:
4302
- logger.warning("Watchdog: limpiando trabajo huérfano %s fuera de cola.", job.job_id)
4303
- await _force_finish_job(job, "❌ **Trabajo eliminado por limpieza automática.**")
4304
- continue
4305
-
4306
- if queue_pos is not None and queue_pos[1] > 0:
4307
- job.is_processing = False
4308
- job.queue_name = queue_pos[0]
4309
- continue
4310
-
4311
- if proc_running:
4312
- ffmpeg_progress_at = job.ffmpeg_last_progress_at or job.last_activity_at or now
4313
- ffmpeg_output_at = job.ffmpeg_last_output_at or ffmpeg_progress_at
4314
- progress_idle = now - ffmpeg_progress_at
4315
- output_idle = now - ffmpeg_output_at
4316
- cpu_busy = (job.cpu_usage_pct or 0.0) >= STALL_CPU_FLOOR_PERCENT
4317
- watchdog_hard_limit = max(stale_limit, COMPRESS_STALL_SECONDS + FFMPEG_OUTPUT_IDLE_SECONDS + 120)
4318
- if progress_idle > watchdog_hard_limit and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and not cpu_busy:
4319
- logger.warning("Watchdog: trabajo %s cancelado por inactividad real (progress=%ss output=%ss).", job.job_id, int(progress_idle), int(output_idle))
4320
- reason = "❌ **Trabajo cancelado por inactividad real.**\n\nFFmpeg dejó de dar progreso y salida durante demasiado tiempo.\nReenvía el video para intentarlo otra vez."
4321
- if job.ffmpeg_last_stderr:
4322
- reason += f"\n\nÚltimo mensaje FFmpeg:\n`{job.ffmpeg_last_stderr[:160]}`"
4323
- await _force_finish_job(job, reason)
4324
- continue
4325
-
4326
- if job.is_processing and idle_for > stale_limit:
4327
- logger.warning("Watchdog: trabajo %s atascado en etapa %s sin proceso activo.", job.job_id, job.stage or "desconocida")
4328
- if job.task and not task_done:
4329
- with contextlib.suppress(Exception):
4330
- job.task.cancel()
4331
- await _force_finish_job(
4332
- job,
4333
- "❌ **Trabajo atascado y limpiado automáticamente.**\n\nLa cola quedó liberada. Puedes reenviar el video.",
4334
- )
4335
- if _global_currently_processing == job.job_id:
4336
- _global_currently_processing = None
4337
- for uq in _user_queues.values():
4338
- if uq.get("processing") == job.job_id:
4339
- uq["processing"] = None
4340
-
4341
- if task_done and not proc_running and idle_for > 10:
4342
- logger.warning("Watchdog: limpiando trabajo %s con task finalizada pero aún registrado.", job.job_id)
4343
- await _force_finish_job(job, "❌ **Trabajo finalizado con estado inconsistente y fue limpiado.**")
4344
  except Exception as e:
4345
  logger.error("Queue watchdog error: %s", e)
4346
 
4347
- @bot.on_message(filters.command("version") & (filters.private | filters.group))
4348
- async def cmd_version(client, msg: Message):
4349
- await msg.reply_text(
4350
- f"🧾 **Versión del bot**\n\n"
4351
- f"🤖 {BOT_VERSION}\n"
4352
- f"🔒 Modo: `{'Público' if BOT_MODE == 'public' else 'Privado'}`\n"
4353
- f"🧵 Colas por plan: `{'ON' if INDIVIDUAL_QUEUES_ENABLED else 'OFF'}`\n"
4354
- f"🛡️ Tope CPU host: `{MAX_CPU_PERCENT}%`\n"
4355
- f"🧯 Stall watchdog: `{COMPRESS_STALL_SECONDS}s`"
4356
- )
4357
-
4358
-
4359
- @bot.on_message(filters.command("misjobs") & (filters.private | filters.group))
4360
- async def cmd_misjobs(client, msg: Message):
4361
- uid = msg.from_user.id
4362
- jobs = [job for job in ACTIVE_JOBS.values() if job.user_id == uid and not job.cancelled]
4363
- if not jobs:
4364
- await msg.reply_text("📭 **No tienes trabajos activos ni en cola.**")
4365
- return
4366
-
4367
- lines = ["🎞️ **Tus trabajos**", "━━━━━━━━━━━━━━━━━━━━━"]
4368
- for idx, job in enumerate(sorted(jobs, key=lambda j: j.started_at or 0, reverse=True)[:15], 1):
4369
- qpos = _job_queue_position(job.job_id)
4370
- queue_label = _plan_queue_label((qpos[0] if qpos else job.queue_name) or "free")
4371
- pos_txt = "procesando" if qpos and qpos[1] == 0 else (f"#{qpos[1]}/{qpos[2]}" if qpos else "—")
4372
- lines.append(
4373
- f"{idx}. `{job.fname or job.job_id}`\n"
4374
- f" • Etapa: `{job.stage or 'pendiente'}`\n"
4375
- f" • Cola: `{queue_label}` | Posición: `{pos_txt}`\n"
4376
- f" • Progreso: `{job.progress_pct:.1f}%` | Tiempo: `{_job_elapsed(job)}`"
4377
- )
4378
- await _send_long(msg, "\n".join(lines))
4379
-
4380
-
4381
- @bot.on_message(filters.command("limpiaratascos") & filters.private)
4382
- async def cmd_limpiaratascos(client, msg: Message):
4383
- global _global_currently_processing
4384
- if not await is_super_admin(msg.from_user.id):
4385
- await _no_access(msg)
4386
- return
4387
-
4388
- cleaned = []
4389
- now = time.time()
4390
- for job in list(ACTIVE_JOBS.values()):
4391
- qpos = _job_queue_position(job.job_id)
4392
- idle_for = now - (job.last_activity_at or now)
4393
- proc_running = bool(job.process and job.process.returncode is None)
4394
- if job.cancelled or (job.is_processing and not proc_running and idle_for > 30) or (qpos is None and idle_for > 30):
4395
- cleaned.append(job.job_id)
4396
- await _force_finish_job(job, "❌ **Trabajo limpiado manualmente por administración.**")
4397
- if _global_currently_processing == job.job_id:
4398
- _global_currently_processing = None
4399
- for uq in _user_queues.values():
4400
- if uq.get("processing") == job.job_id:
4401
- uq["processing"] = None
4402
-
4403
- if _global_queue_list:
4404
- _global_queue_event.set()
4405
- for uq in _user_queues.values():
4406
- if uq.get("list"):
4407
- uq["event"].set()
4408
-
4409
- if not cleaned:
4410
- await msg.reply_text("✅ **No encontré trabajos atascados para limpiar.**")
4411
- return
4412
- await msg.reply_text("✅ **Limpieza completada**\n\n" + "\n".join(f"• `{jid}`" for jid in cleaned[:30]))
4413
-
4414
-
4415
  # ========================
4416
  # COMANDOS DE MENÚ
4417
  # ========================
@@ -4427,8 +4222,6 @@ USER_COMMANDS = [
4427
  ("cancelar", "❌ Cancelar tu compresión activa"),
4428
  ("reporte", "📝 Reportar un problema"),
4429
  ("ping", "🏓 Verificar si el bot responde"),
4430
- ("misjobs", "🎞️ Ver tus trabajos activos"),
4431
- ("version", "🧾 Ver versión y modo del bot"),
4432
  ("velocidad", "⚡ Test de velocidad del servidor"),
4433
  ("id", "🆔 Ver tu ID"),
4434
  ("about", "ℹ️ Información del bot"),
@@ -4475,7 +4268,6 @@ SUPERADMIN_COMMANDS = ADMIN_COMMANDS + [
4475
  ("importdb", "📥 Importar backup DB"),
4476
  ("logs", "📡 Configurar canal de logs"),
4477
  ("forzarwatchdog", "🛠️ Reparar workers/colas atascadas"),
4478
- ("limpiaratascos", "🧹 Limpiar trabajos colgados"),
4479
  ("actualizarmenu", "🔄 Actualizar menú de comandos"),
4480
  ]
4481
 
@@ -4550,7 +4342,6 @@ async def start_bot():
4550
  f"🔒 Modo: {BOT_MODE} | 🎬 Log videos: {LOG_VIDEOS}\n"
4551
  f"🧵 Colas por plan: {INDIVIDUAL_QUEUES_ENABLED}"
4552
  )
4553
- await _ensure_global_worker()
4554
  asyncio.create_task(_plan_expiry_checker())
4555
  asyncio.create_task(_queue_watchdog())
4556
  asyncio.create_task(_set_menus())
 
158
  PROGRESS_EDIT_EVERY = 5
159
  MAX_CPU_PERCENT = max(50, min(95, int(os.environ.get("MAX_CPU_PERCENT", "85"))))
160
  CPU_MONITOR_INTERVAL = max(0.5, float(os.environ.get("CPU_MONITOR_INTERVAL", "2.0")))
161
+ COMPRESS_STALL_SECONDS = max(60, int(os.environ.get("COMPRESS_STALL_SECONDS", "180")))
 
 
 
162
  QUEUE_WATCHDOG_INTERVAL = max(2.0, float(os.environ.get("QUEUE_WATCHDOG_INTERVAL", "5.0")))
163
  CPU_LIMIT_HEADROOM_PERCENT = max(1, min(15, int(os.environ.get("CPU_LIMIT_HEADROOM_PERCENT", "4"))))
164
 
 
980
  cpu_usage_pct: float = 0.0
981
  cpu_slots: int = 0
982
  cpu_slots_max: int = 0
 
 
 
983
 
984
  ACTIVE_JOBS: Dict[str, Job] = {}
985
 
 
1122
  return True
1123
  return any(uq.get("processing") == job_id for uq in _user_queues.values())
1124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1125
  # =====================
1126
  # UTILS
1127
  # =====================
 
1273
  job.cpu_slots = current_cpu_slots
1274
  job.cpu_slots_max = max_parallel_slots
1275
  job.cpu_usage_pct = 0.0
 
 
 
1276
  _job_touch(job, stage="comprimiendo", progress=0)
1277
 
1278
  out_time_ms = 0
1279
+ last_progress_at = time.time()
 
 
 
1280
  stderr_tail = deque(maxlen=80)
1281
+ t0 = time.time()
1282
  ffmpeg_cpu_pct_total = 0.0
1283
  ffmpeg_cpu_pct_quota = 0.0
 
 
 
 
 
 
1284
 
1285
  async def _read_stdout():
1286
+ nonlocal out_time_ms, last_progress_at
1287
  async for line in proc.stdout:
 
 
 
1288
  s = line.decode("utf-8", errors="replace").strip()
1289
  if "=" not in s:
1290
  continue
 
1295
  if new_out_time_ms >= out_time_ms:
1296
  out_time_ms = new_out_time_ms
1297
  last_progress_at = time.time()
 
 
1298
  except Exception:
1299
  pass
1300
  elif k == "progress" and v == "end":
1301
  last_progress_at = time.time()
 
 
1302
  break
1303
 
1304
  async def _read_stderr():
 
1305
  while True:
1306
  line = await proc.stderr.readline()
1307
  if not line:
1308
  break
1309
+ stderr_tail.append(line.decode("utf-8", errors="replace").strip())
 
 
 
 
 
 
1310
 
1311
  async def _cpu_guard():
1312
+ nonlocal ffmpeg_cpu_pct_total, ffmpeg_cpu_pct_quota, current_cpu_slots, cpu_subset, active_jobs
1313
  if proc.pid is None:
1314
  return
1315
  try:
 
1350
  job.cpu_usage_pct = ffmpeg_cpu_pct_total
1351
  job.cpu_slots = current_cpu_slots
1352
  job.cpu_slots_max = max_parallel_slots
 
1353
  _job_touch(job, stage="comprimiendo", progress=(job.progress_pct if job else None))
1354
  except Exception:
1355
  pass
 
1364
  while not wt.done():
1365
  if job and job.cancelled:
1366
  raise asyncio.CancelledError()
1367
+ if (time.time() - last_progress_at) > COMPRESS_STALL_SECONDS:
1368
+ raise RuntimeError("La compresión quedó bloqueada sin progreso durante demasiado tiempo.")
 
 
 
 
 
 
 
 
 
 
1369
  await asyncio.sleep(PROGRESS_EDIT_EVERY)
1370
  ds = out_time_ms / 1_000_000 if out_time_ms else 0.0
1371
  p = (ds / dur) if dur > 0 else 0.0
 
1386
  f"🧠 CPU: `{cpu_slots_txt} cores` | FFmpeg: `{ffmpeg_cpu_pct_total:.1f}%` host / `{ffmpeg_cpu_pct_quota:.1f}%` cuota\n"
1387
  f"🛡️ Tope(host): `{MAX_CPU_PERCENT}%` | Pool: `{max_parallel_slots}` | Jobs: `{active_jobs}`\n"
1388
  f"🧩 Cuota cgroup: `{quota_txt}` CPU | Capacidad: `{cpu_capacity:.2f}`\n"
 
1389
  f"⏳ ETA: `{_fmt_eta(eta)}`"
1390
  )
1391
  await wt
 
1408
 
1409
  if proc.returncode != 0:
1410
  err = "\n".join([x for x in stderr_tail if x][-8:]) or "FFmpeg terminó con error."
 
 
 
 
1411
  raise RuntimeError(err)
1412
  if not os.path.exists(out_f) or os.path.getsize(out_f) == 0:
1413
  raise RuntimeError("No se generó el archivo comprimido.")
 
1545
  # PROCESO PRINCIPAL
1546
  # =====================
1547
  async def _process(client, message: Message, status_msg: Message = None, job: Job = None):
 
 
1548
  editor = _Editor(status_msg, job_id=(job.job_id if job else None))
1549
  uid = message.from_user.id
1550
  cid = message.chat.id
 
1706
  pass
1707
  finally:
1708
  if job:
 
1709
  await _cleanup_job(job)
1710
  else:
1711
  for p in (in_f, out_f, thumb):
 
3590
 
3591
  lines = ["🧰 **Trabajos del bot**", "━━━━━━━━━━━━━━━━━━━━━"]
3592
  for idx, job in enumerate(sorted(active, key=lambda j: (j.is_processing, j.started_at or 0), reverse=True)[:20], 1):
3593
+ state = "🟢 Activo" if job.is_processing else "🟡 En espera"
3594
+ queue_label = _plan_queue_label(job.queue_name or "free") if job.queue_name else "🧵 Cola pendiente"
 
 
 
 
 
3595
  lines.append(
3596
  f"{idx}. {state} | `{job.user_id}` | `{job.fname or job.job_id}`\n"
3597
+ f" • Etapa: `{job.stage or 'pendiente'}` | Cola: `{queue_label}`\n"
3598
  f" • Progreso: `{job.progress_pct:.1f}%` | CPU: `{job.cpu_usage_pct:.1f}%` | Slots: `{job.cpu_slots}/{job.cpu_slots_max}`\n"
3599
  f" • Última actividad: `{_fmt_eta(max(0, time.time() - (job.last_activity_at or time.time())))}` atrás | Tiempo: `{_job_elapsed(job)}`"
3600
  )
 
4179
  if _global_worker_task and _global_worker_task.done():
4180
  logger.warning("Watchdog: reiniciando worker global con %d elemento(s) pendientes.", len(_global_queue_list))
4181
  await _ensure_global_worker()
 
4182
 
4183
  for queue_key, uq in list(_user_queues.items()):
4184
  processing_id = uq.get("processing")
 
4190
  if worker and worker.done():
4191
  logger.warning("Watchdog: reiniciando worker %s con %d elemento(s) pendientes.", queue_key, len(uq["list"]))
4192
  await _ensure_user_worker(queue_key)
 
4193
 
4194
  now = time.time()
 
4195
  for job in list(ACTIVE_JOBS.values()):
4196
+ if not job.is_processing or job.cancelled:
 
4197
  continue
4198
+ if job.process and job.process.returncode is None:
4199
+ idle_for = now - (job.last_activity_at or now)
4200
+ if idle_for > (COMPRESS_STALL_SECONDS + 30):
4201
+ logger.warning("Watchdog: trabajo %s cancelado por inactividad (%ss).", job.job_id, int(idle_for))
4202
+ job.cancelled = True
4203
+ await _terminate_process(job.process)
4204
+ elif job.stage == "comprimiendo" and (now - (job.last_activity_at or now)) > (COMPRESS_STALL_SECONDS + 30):
4205
+ logger.warning("Watchdog: trabajo %s quedó sin proceso activo en compresión; se marca cancelado.", job.job_id)
4206
+ job.cancelled = True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4207
  except Exception as e:
4208
  logger.error("Queue watchdog error: %s", e)
4209
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4210
  # ========================
4211
  # COMANDOS DE MENÚ
4212
  # ========================
 
4222
  ("cancelar", "❌ Cancelar tu compresión activa"),
4223
  ("reporte", "📝 Reportar un problema"),
4224
  ("ping", "🏓 Verificar si el bot responde"),
 
 
4225
  ("velocidad", "⚡ Test de velocidad del servidor"),
4226
  ("id", "🆔 Ver tu ID"),
4227
  ("about", "ℹ️ Información del bot"),
 
4268
  ("importdb", "📥 Importar backup DB"),
4269
  ("logs", "📡 Configurar canal de logs"),
4270
  ("forzarwatchdog", "🛠️ Reparar workers/colas atascadas"),
 
4271
  ("actualizarmenu", "🔄 Actualizar menú de comandos"),
4272
  ]
4273
 
 
4342
  f"🔒 Modo: {BOT_MODE} | 🎬 Log videos: {LOG_VIDEOS}\n"
4343
  f"🧵 Colas por plan: {INDIVIDUAL_QUEUES_ENABLED}"
4344
  )
 
4345
  asyncio.create_task(_plan_expiry_checker())
4346
  asyncio.create_task(_queue_watchdog())
4347
  asyncio.create_task(_set_menus())