Spaces:
Paused
Paused
Upload main.py
Browse files
main.py
CHANGED
|
@@ -158,10 +158,7 @@ VIDEO_EXTENSIONS = {
|
|
| 158 |
PROGRESS_EDIT_EVERY = 5
|
| 159 |
MAX_CPU_PERCENT = max(50, min(95, int(os.environ.get("MAX_CPU_PERCENT", "85"))))
|
| 160 |
CPU_MONITOR_INTERVAL = max(0.5, float(os.environ.get("CPU_MONITOR_INTERVAL", "2.0")))
|
| 161 |
-
COMPRESS_STALL_SECONDS = max(
|
| 162 |
-
COMPRESS_START_GRACE_SECONDS = max(60, int(os.environ.get("COMPRESS_START_GRACE_SECONDS", "120")))
|
| 163 |
-
FFMPEG_OUTPUT_IDLE_SECONDS = max(45, int(os.environ.get("FFMPEG_OUTPUT_IDLE_SECONDS", "120")))
|
| 164 |
-
STALL_CPU_FLOOR_PERCENT = max(0.5, float(os.environ.get("STALL_CPU_FLOOR_PERCENT", "2.0")))
|
| 165 |
QUEUE_WATCHDOG_INTERVAL = max(2.0, float(os.environ.get("QUEUE_WATCHDOG_INTERVAL", "5.0")))
|
| 166 |
CPU_LIMIT_HEADROOM_PERCENT = max(1, min(15, int(os.environ.get("CPU_LIMIT_HEADROOM_PERCENT", "4"))))
|
| 167 |
|
|
@@ -983,9 +980,6 @@ class Job:
|
|
| 983 |
cpu_usage_pct: float = 0.0
|
| 984 |
cpu_slots: int = 0
|
| 985 |
cpu_slots_max: int = 0
|
| 986 |
-
ffmpeg_last_progress_at: float = 0.0
|
| 987 |
-
ffmpeg_last_output_at: float = 0.0
|
| 988 |
-
ffmpeg_last_stderr: str = ""
|
| 989 |
|
| 990 |
ACTIVE_JOBS: Dict[str, Job] = {}
|
| 991 |
|
|
@@ -1128,43 +1122,6 @@ def _is_job_processing(job_id: str) -> bool:
|
|
| 1128 |
return True
|
| 1129 |
return any(uq.get("processing") == job_id for uq in _user_queues.values())
|
| 1130 |
|
| 1131 |
-
|
| 1132 |
-
def _job_queue_position(job_id: str) -> Optional[tuple[str, int, int]]:
|
| 1133 |
-
if _global_currently_processing == job_id:
|
| 1134 |
-
return ("free", 0, max(1, len(_global_queue_list) + 1))
|
| 1135 |
-
for i, item in enumerate(_global_queue_list, 1):
|
| 1136 |
-
if item[3] == job_id:
|
| 1137 |
-
return ("free", i, len(_global_queue_list))
|
| 1138 |
-
for queue_key, uq in _user_queues.items():
|
| 1139 |
-
if uq.get("processing") == job_id:
|
| 1140 |
-
total = len(uq.get("list", [])) + 1
|
| 1141 |
-
return (queue_key, 0, max(1, total))
|
| 1142 |
-
for i, item in enumerate(uq.get("list", []), 1):
|
| 1143 |
-
if item[3] == job_id:
|
| 1144 |
-
total = len(uq.get("list", [])) + (1 if uq.get("processing") else 0)
|
| 1145 |
-
return (queue_key, i, max(1, total))
|
| 1146 |
-
return None
|
| 1147 |
-
|
| 1148 |
-
|
| 1149 |
-
async def _edit_job_status(job: Job, text: str, *, remove_markup: bool = True):
|
| 1150 |
-
try:
|
| 1151 |
-
await bot.edit_message_text(
|
| 1152 |
-
chat_id=job.chat_id,
|
| 1153 |
-
message_id=job.status_msg_id,
|
| 1154 |
-
text=text,
|
| 1155 |
-
reply_markup=None if remove_markup else _cancel_markup(job.job_id),
|
| 1156 |
-
)
|
| 1157 |
-
except Exception:
|
| 1158 |
-
pass
|
| 1159 |
-
|
| 1160 |
-
|
| 1161 |
-
async def _force_finish_job(job: Job, reason: str, *, notify_user: bool = True):
|
| 1162 |
-
job.cancelled = True
|
| 1163 |
-
job.is_processing = False
|
| 1164 |
-
if notify_user:
|
| 1165 |
-
await _edit_job_status(job, reason, remove_markup=True)
|
| 1166 |
-
await _cleanup_job(job)
|
| 1167 |
-
|
| 1168 |
# =====================
|
| 1169 |
# UTILS
|
| 1170 |
# =====================
|
|
@@ -1316,33 +1273,18 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1316 |
job.cpu_slots = current_cpu_slots
|
| 1317 |
job.cpu_slots_max = max_parallel_slots
|
| 1318 |
job.cpu_usage_pct = 0.0
|
| 1319 |
-
job.ffmpeg_last_progress_at = time.time()
|
| 1320 |
-
job.ffmpeg_last_output_at = time.time()
|
| 1321 |
-
job.ffmpeg_last_stderr = ""
|
| 1322 |
_job_touch(job, stage="comprimiendo", progress=0)
|
| 1323 |
|
| 1324 |
out_time_ms = 0
|
| 1325 |
-
|
| 1326 |
-
last_progress_at = now0
|
| 1327 |
-
last_output_at = now0
|
| 1328 |
-
last_stderr_at = now0
|
| 1329 |
stderr_tail = deque(maxlen=80)
|
| 1330 |
-
t0 =
|
| 1331 |
ffmpeg_cpu_pct_total = 0.0
|
| 1332 |
ffmpeg_cpu_pct_quota = 0.0
|
| 1333 |
-
proc_cpu_raw = 0.0
|
| 1334 |
-
input_size_mb = 0.0
|
| 1335 |
-
with contextlib.suppress(Exception):
|
| 1336 |
-
input_size_mb = max(0.0, os.path.getsize(in_f) / (1024 * 1024))
|
| 1337 |
-
dynamic_start_grace = max(COMPRESS_START_GRACE_SECONDS, min(300, int(input_size_mb * 0.35) + int(dur * 0.02 if dur else 0)))
|
| 1338 |
-
dynamic_stall_limit = max(COMPRESS_STALL_SECONDS, min(1200, int(dur * 0.12) if dur else 0))
|
| 1339 |
|
| 1340 |
async def _read_stdout():
|
| 1341 |
-
nonlocal out_time_ms, last_progress_at
|
| 1342 |
async for line in proc.stdout:
|
| 1343 |
-
last_output_at = time.time()
|
| 1344 |
-
if job:
|
| 1345 |
-
job.ffmpeg_last_output_at = last_output_at
|
| 1346 |
s = line.decode("utf-8", errors="replace").strip()
|
| 1347 |
if "=" not in s:
|
| 1348 |
continue
|
|
@@ -1353,32 +1295,21 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1353 |
if new_out_time_ms >= out_time_ms:
|
| 1354 |
out_time_ms = new_out_time_ms
|
| 1355 |
last_progress_at = time.time()
|
| 1356 |
-
if job:
|
| 1357 |
-
job.ffmpeg_last_progress_at = last_progress_at
|
| 1358 |
except Exception:
|
| 1359 |
pass
|
| 1360 |
elif k == "progress" and v == "end":
|
| 1361 |
last_progress_at = time.time()
|
| 1362 |
-
if job:
|
| 1363 |
-
job.ffmpeg_last_progress_at = last_progress_at
|
| 1364 |
break
|
| 1365 |
|
| 1366 |
async def _read_stderr():
|
| 1367 |
-
nonlocal last_output_at, last_stderr_at
|
| 1368 |
while True:
|
| 1369 |
line = await proc.stderr.readline()
|
| 1370 |
if not line:
|
| 1371 |
break
|
| 1372 |
-
|
| 1373 |
-
last_stderr_at = last_output_at
|
| 1374 |
-
decoded = line.decode("utf-8", errors="replace").strip()
|
| 1375 |
-
stderr_tail.append(decoded)
|
| 1376 |
-
if job:
|
| 1377 |
-
job.ffmpeg_last_output_at = last_output_at
|
| 1378 |
-
job.ffmpeg_last_stderr = decoded[:300]
|
| 1379 |
|
| 1380 |
async def _cpu_guard():
|
| 1381 |
-
nonlocal ffmpeg_cpu_pct_total, ffmpeg_cpu_pct_quota, current_cpu_slots, cpu_subset, active_jobs
|
| 1382 |
if proc.pid is None:
|
| 1383 |
return
|
| 1384 |
try:
|
|
@@ -1419,7 +1350,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1419 |
job.cpu_usage_pct = ffmpeg_cpu_pct_total
|
| 1420 |
job.cpu_slots = current_cpu_slots
|
| 1421 |
job.cpu_slots_max = max_parallel_slots
|
| 1422 |
-
job.ffmpeg_last_output_at = max(job.ffmpeg_last_output_at or 0.0, last_output_at)
|
| 1423 |
_job_touch(job, stage="comprimiendo", progress=(job.progress_pct if job else None))
|
| 1424 |
except Exception:
|
| 1425 |
pass
|
|
@@ -1434,18 +1364,8 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1434 |
while not wt.done():
|
| 1435 |
if job and job.cancelled:
|
| 1436 |
raise asyncio.CancelledError()
|
| 1437 |
-
|
| 1438 |
-
|
| 1439 |
-
progress_idle = now - last_progress_at
|
| 1440 |
-
output_idle = now - max(last_output_at, last_stderr_at, last_progress_at)
|
| 1441 |
-
no_progress_yet = out_time_ms <= 0
|
| 1442 |
-
cpu_looks_idle = proc_cpu_raw < STALL_CPU_FLOOR_PERCENT
|
| 1443 |
-
if no_progress_yet:
|
| 1444 |
-
if elapsed > dynamic_start_grace and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and cpu_looks_idle:
|
| 1445 |
-
raise RuntimeError("FFmpeg no arrancó bien la compresión y quedó sin actividad.")
|
| 1446 |
-
else:
|
| 1447 |
-
if progress_idle > dynamic_stall_limit and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and cpu_looks_idle:
|
| 1448 |
-
raise RuntimeError("La compresión quedó atascada sin avance real durante demasiado tiempo.")
|
| 1449 |
await asyncio.sleep(PROGRESS_EDIT_EVERY)
|
| 1450 |
ds = out_time_ms / 1_000_000 if out_time_ms else 0.0
|
| 1451 |
p = (ds / dur) if dur > 0 else 0.0
|
|
@@ -1466,7 +1386,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1466 |
f"🧠 CPU: `{cpu_slots_txt} cores` | FFmpeg: `{ffmpeg_cpu_pct_total:.1f}%` host / `{ffmpeg_cpu_pct_quota:.1f}%` cuota\n"
|
| 1467 |
f"🛡️ Tope(host): `{MAX_CPU_PERCENT}%` | Pool: `{max_parallel_slots}` | Jobs: `{active_jobs}`\n"
|
| 1468 |
f"🧩 Cuota cgroup: `{quota_txt}` CPU | Capacidad: `{cpu_capacity:.2f}`\n"
|
| 1469 |
-
f"🧯 Stall: `{int(progress_idle)}s/{dynamic_stall_limit}s` | Salida: `{int(output_idle)}s/{FFMPEG_OUTPUT_IDLE_SECONDS}s`\n"
|
| 1470 |
f"⏳ ETA: `{_fmt_eta(eta)}`"
|
| 1471 |
)
|
| 1472 |
await wt
|
|
@@ -1489,10 +1408,6 @@ async def _compress(in_f, out_f, uid, status_msg=None, job=None, title=""):
|
|
| 1489 |
|
| 1490 |
if proc.returncode != 0:
|
| 1491 |
err = "\n".join([x for x in stderr_tail if x][-8:]) or "FFmpeg terminó con error."
|
| 1492 |
-
if proc.returncode in (-9, 137):
|
| 1493 |
-
err = "El proceso de compresión fue terminado por el sistema (memoria/recursos insuficientes)."
|
| 1494 |
-
elif proc.returncode in (-15, 143) and not (job and job.cancelled):
|
| 1495 |
-
err = "La compresión fue detenida antes de terminar. Revisa watchdog, reinicios del hosting o falta de recursos."
|
| 1496 |
raise RuntimeError(err)
|
| 1497 |
if not os.path.exists(out_f) or os.path.getsize(out_f) == 0:
|
| 1498 |
raise RuntimeError("No se generó el archivo comprimido.")
|
|
@@ -1630,8 +1545,6 @@ async def _user_worker(queue_key: str):
|
|
| 1630 |
# PROCESO PRINCIPAL
|
| 1631 |
# =====================
|
| 1632 |
async def _process(client, message: Message, status_msg: Message = None, job: Job = None):
|
| 1633 |
-
if job:
|
| 1634 |
-
job.task = asyncio.current_task()
|
| 1635 |
editor = _Editor(status_msg, job_id=(job.job_id if job else None))
|
| 1636 |
uid = message.from_user.id
|
| 1637 |
cid = message.chat.id
|
|
@@ -1793,7 +1706,6 @@ async def _process(client, message: Message, status_msg: Message = None, job: Jo
|
|
| 1793 |
pass
|
| 1794 |
finally:
|
| 1795 |
if job:
|
| 1796 |
-
job.task = None
|
| 1797 |
await _cleanup_job(job)
|
| 1798 |
else:
|
| 1799 |
for p in (in_f, out_f, thumb):
|
|
@@ -3678,16 +3590,11 @@ async def cmd_trabajos(client, msg: Message):
|
|
| 3678 |
|
| 3679 |
lines = ["🧰 **Trabajos del bot**", "━━━━━━━━━━━━━━━━━━━━━"]
|
| 3680 |
for idx, job in enumerate(sorted(active, key=lambda j: (j.is_processing, j.started_at or 0), reverse=True)[:20], 1):
|
| 3681 |
-
|
| 3682 |
-
|
| 3683 |
-
state = "🟢 Activo" if really_processing else "🟡 En espera"
|
| 3684 |
-
queue_label = _plan_queue_label((qpos[0] if qpos else job.queue_name) or "free") if (qpos or job.queue_name) else "🧵 Cola pendiente"
|
| 3685 |
-
extra_queue = ""
|
| 3686 |
-
if qpos and qpos[1] > 0:
|
| 3687 |
-
extra_queue = f" | Pos: `#{qpos[1]}/{qpos[2]}`"
|
| 3688 |
lines.append(
|
| 3689 |
f"{idx}. {state} | `{job.user_id}` | `{job.fname or job.job_id}`\n"
|
| 3690 |
-
f" • Etapa: `{job.stage or 'pendiente'}` | Cola: `{queue_label}`
|
| 3691 |
f" • Progreso: `{job.progress_pct:.1f}%` | CPU: `{job.cpu_usage_pct:.1f}%` | Slots: `{job.cpu_slots}/{job.cpu_slots_max}`\n"
|
| 3692 |
f" • Última actividad: `{_fmt_eta(max(0, time.time() - (job.last_activity_at or time.time())))}` atrás | Tiempo: `{_job_elapsed(job)}`"
|
| 3693 |
)
|
|
@@ -4272,7 +4179,6 @@ async def _queue_watchdog():
|
|
| 4272 |
if _global_worker_task and _global_worker_task.done():
|
| 4273 |
logger.warning("Watchdog: reiniciando worker global con %d elemento(s) pendientes.", len(_global_queue_list))
|
| 4274 |
await _ensure_global_worker()
|
| 4275 |
-
_global_queue_event.set()
|
| 4276 |
|
| 4277 |
for queue_key, uq in list(_user_queues.items()):
|
| 4278 |
processing_id = uq.get("processing")
|
|
@@ -4284,134 +4190,23 @@ async def _queue_watchdog():
|
|
| 4284 |
if worker and worker.done():
|
| 4285 |
logger.warning("Watchdog: reiniciando worker %s con %d elemento(s) pendientes.", queue_key, len(uq["list"]))
|
| 4286 |
await _ensure_user_worker(queue_key)
|
| 4287 |
-
uq["event"].set()
|
| 4288 |
|
| 4289 |
now = time.time()
|
| 4290 |
-
stale_limit = COMPRESS_STALL_SECONDS + 30
|
| 4291 |
for job in list(ACTIVE_JOBS.values()):
|
| 4292 |
-
if job.cancelled:
|
| 4293 |
-
await _force_finish_job(job, "❌ **Trabajo cancelado por watchdog.**")
|
| 4294 |
continue
|
| 4295 |
-
|
| 4296 |
-
|
| 4297 |
-
|
| 4298 |
-
|
| 4299 |
-
|
| 4300 |
-
|
| 4301 |
-
|
| 4302 |
-
logger.warning("Watchdog:
|
| 4303 |
-
|
| 4304 |
-
continue
|
| 4305 |
-
|
| 4306 |
-
if queue_pos is not None and queue_pos[1] > 0:
|
| 4307 |
-
job.is_processing = False
|
| 4308 |
-
job.queue_name = queue_pos[0]
|
| 4309 |
-
continue
|
| 4310 |
-
|
| 4311 |
-
if proc_running:
|
| 4312 |
-
ffmpeg_progress_at = job.ffmpeg_last_progress_at or job.last_activity_at or now
|
| 4313 |
-
ffmpeg_output_at = job.ffmpeg_last_output_at or ffmpeg_progress_at
|
| 4314 |
-
progress_idle = now - ffmpeg_progress_at
|
| 4315 |
-
output_idle = now - ffmpeg_output_at
|
| 4316 |
-
cpu_busy = (job.cpu_usage_pct or 0.0) >= STALL_CPU_FLOOR_PERCENT
|
| 4317 |
-
watchdog_hard_limit = max(stale_limit, COMPRESS_STALL_SECONDS + FFMPEG_OUTPUT_IDLE_SECONDS + 120)
|
| 4318 |
-
if progress_idle > watchdog_hard_limit and output_idle > FFMPEG_OUTPUT_IDLE_SECONDS and not cpu_busy:
|
| 4319 |
-
logger.warning("Watchdog: trabajo %s cancelado por inactividad real (progress=%ss output=%ss).", job.job_id, int(progress_idle), int(output_idle))
|
| 4320 |
-
reason = "❌ **Trabajo cancelado por inactividad real.**\n\nFFmpeg dejó de dar progreso y salida durante demasiado tiempo.\nReenvía el video para intentarlo otra vez."
|
| 4321 |
-
if job.ffmpeg_last_stderr:
|
| 4322 |
-
reason += f"\n\nÚltimo mensaje FFmpeg:\n`{job.ffmpeg_last_stderr[:160]}`"
|
| 4323 |
-
await _force_finish_job(job, reason)
|
| 4324 |
-
continue
|
| 4325 |
-
|
| 4326 |
-
if job.is_processing and idle_for > stale_limit:
|
| 4327 |
-
logger.warning("Watchdog: trabajo %s atascado en etapa %s sin proceso activo.", job.job_id, job.stage or "desconocida")
|
| 4328 |
-
if job.task and not task_done:
|
| 4329 |
-
with contextlib.suppress(Exception):
|
| 4330 |
-
job.task.cancel()
|
| 4331 |
-
await _force_finish_job(
|
| 4332 |
-
job,
|
| 4333 |
-
"❌ **Trabajo atascado y limpiado automáticamente.**\n\nLa cola quedó liberada. Puedes reenviar el video.",
|
| 4334 |
-
)
|
| 4335 |
-
if _global_currently_processing == job.job_id:
|
| 4336 |
-
_global_currently_processing = None
|
| 4337 |
-
for uq in _user_queues.values():
|
| 4338 |
-
if uq.get("processing") == job.job_id:
|
| 4339 |
-
uq["processing"] = None
|
| 4340 |
-
|
| 4341 |
-
if task_done and not proc_running and idle_for > 10:
|
| 4342 |
-
logger.warning("Watchdog: limpiando trabajo %s con task finalizada pero aún registrado.", job.job_id)
|
| 4343 |
-
await _force_finish_job(job, "❌ **Trabajo finalizado con estado inconsistente y fue limpiado.**")
|
| 4344 |
except Exception as e:
|
| 4345 |
logger.error("Queue watchdog error: %s", e)
|
| 4346 |
|
| 4347 |
-
@bot.on_message(filters.command("version") & (filters.private | filters.group))
|
| 4348 |
-
async def cmd_version(client, msg: Message):
|
| 4349 |
-
await msg.reply_text(
|
| 4350 |
-
f"🧾 **Versión del bot**\n\n"
|
| 4351 |
-
f"🤖 {BOT_VERSION}\n"
|
| 4352 |
-
f"🔒 Modo: `{'Público' if BOT_MODE == 'public' else 'Privado'}`\n"
|
| 4353 |
-
f"🧵 Colas por plan: `{'ON' if INDIVIDUAL_QUEUES_ENABLED else 'OFF'}`\n"
|
| 4354 |
-
f"🛡️ Tope CPU host: `{MAX_CPU_PERCENT}%`\n"
|
| 4355 |
-
f"🧯 Stall watchdog: `{COMPRESS_STALL_SECONDS}s`"
|
| 4356 |
-
)
|
| 4357 |
-
|
| 4358 |
-
|
| 4359 |
-
@bot.on_message(filters.command("misjobs") & (filters.private | filters.group))
|
| 4360 |
-
async def cmd_misjobs(client, msg: Message):
|
| 4361 |
-
uid = msg.from_user.id
|
| 4362 |
-
jobs = [job for job in ACTIVE_JOBS.values() if job.user_id == uid and not job.cancelled]
|
| 4363 |
-
if not jobs:
|
| 4364 |
-
await msg.reply_text("📭 **No tienes trabajos activos ni en cola.**")
|
| 4365 |
-
return
|
| 4366 |
-
|
| 4367 |
-
lines = ["🎞️ **Tus trabajos**", "━━━━━━━━━━━━━━━━━━━━━"]
|
| 4368 |
-
for idx, job in enumerate(sorted(jobs, key=lambda j: j.started_at or 0, reverse=True)[:15], 1):
|
| 4369 |
-
qpos = _job_queue_position(job.job_id)
|
| 4370 |
-
queue_label = _plan_queue_label((qpos[0] if qpos else job.queue_name) or "free")
|
| 4371 |
-
pos_txt = "procesando" if qpos and qpos[1] == 0 else (f"#{qpos[1]}/{qpos[2]}" if qpos else "—")
|
| 4372 |
-
lines.append(
|
| 4373 |
-
f"{idx}. `{job.fname or job.job_id}`\n"
|
| 4374 |
-
f" • Etapa: `{job.stage or 'pendiente'}`\n"
|
| 4375 |
-
f" • Cola: `{queue_label}` | Posición: `{pos_txt}`\n"
|
| 4376 |
-
f" • Progreso: `{job.progress_pct:.1f}%` | Tiempo: `{_job_elapsed(job)}`"
|
| 4377 |
-
)
|
| 4378 |
-
await _send_long(msg, "\n".join(lines))
|
| 4379 |
-
|
| 4380 |
-
|
| 4381 |
-
@bot.on_message(filters.command("limpiaratascos") & filters.private)
|
| 4382 |
-
async def cmd_limpiaratascos(client, msg: Message):
|
| 4383 |
-
global _global_currently_processing
|
| 4384 |
-
if not await is_super_admin(msg.from_user.id):
|
| 4385 |
-
await _no_access(msg)
|
| 4386 |
-
return
|
| 4387 |
-
|
| 4388 |
-
cleaned = []
|
| 4389 |
-
now = time.time()
|
| 4390 |
-
for job in list(ACTIVE_JOBS.values()):
|
| 4391 |
-
qpos = _job_queue_position(job.job_id)
|
| 4392 |
-
idle_for = now - (job.last_activity_at or now)
|
| 4393 |
-
proc_running = bool(job.process and job.process.returncode is None)
|
| 4394 |
-
if job.cancelled or (job.is_processing and not proc_running and idle_for > 30) or (qpos is None and idle_for > 30):
|
| 4395 |
-
cleaned.append(job.job_id)
|
| 4396 |
-
await _force_finish_job(job, "❌ **Trabajo limpiado manualmente por administración.**")
|
| 4397 |
-
if _global_currently_processing == job.job_id:
|
| 4398 |
-
_global_currently_processing = None
|
| 4399 |
-
for uq in _user_queues.values():
|
| 4400 |
-
if uq.get("processing") == job.job_id:
|
| 4401 |
-
uq["processing"] = None
|
| 4402 |
-
|
| 4403 |
-
if _global_queue_list:
|
| 4404 |
-
_global_queue_event.set()
|
| 4405 |
-
for uq in _user_queues.values():
|
| 4406 |
-
if uq.get("list"):
|
| 4407 |
-
uq["event"].set()
|
| 4408 |
-
|
| 4409 |
-
if not cleaned:
|
| 4410 |
-
await msg.reply_text("✅ **No encontré trabajos atascados para limpiar.**")
|
| 4411 |
-
return
|
| 4412 |
-
await msg.reply_text("✅ **Limpieza completada**\n\n" + "\n".join(f"• `{jid}`" for jid in cleaned[:30]))
|
| 4413 |
-
|
| 4414 |
-
|
| 4415 |
# ========================
|
| 4416 |
# COMANDOS DE MENÚ
|
| 4417 |
# ========================
|
|
@@ -4427,8 +4222,6 @@ USER_COMMANDS = [
|
|
| 4427 |
("cancelar", "❌ Cancelar tu compresión activa"),
|
| 4428 |
("reporte", "📝 Reportar un problema"),
|
| 4429 |
("ping", "🏓 Verificar si el bot responde"),
|
| 4430 |
-
("misjobs", "🎞️ Ver tus trabajos activos"),
|
| 4431 |
-
("version", "🧾 Ver versión y modo del bot"),
|
| 4432 |
("velocidad", "⚡ Test de velocidad del servidor"),
|
| 4433 |
("id", "🆔 Ver tu ID"),
|
| 4434 |
("about", "ℹ️ Información del bot"),
|
|
@@ -4475,7 +4268,6 @@ SUPERADMIN_COMMANDS = ADMIN_COMMANDS + [
|
|
| 4475 |
("importdb", "📥 Importar backup DB"),
|
| 4476 |
("logs", "📡 Configurar canal de logs"),
|
| 4477 |
("forzarwatchdog", "🛠️ Reparar workers/colas atascadas"),
|
| 4478 |
-
("limpiaratascos", "🧹 Limpiar trabajos colgados"),
|
| 4479 |
("actualizarmenu", "🔄 Actualizar menú de comandos"),
|
| 4480 |
]
|
| 4481 |
|
|
@@ -4550,7 +4342,6 @@ async def start_bot():
|
|
| 4550 |
f"🔒 Modo: {BOT_MODE} | 🎬 Log videos: {LOG_VIDEOS}\n"
|
| 4551 |
f"🧵 Colas por plan: {INDIVIDUAL_QUEUES_ENABLED}"
|
| 4552 |
)
|
| 4553 |
-
await _ensure_global_worker()
|
| 4554 |
asyncio.create_task(_plan_expiry_checker())
|
| 4555 |
asyncio.create_task(_queue_watchdog())
|
| 4556 |
asyncio.create_task(_set_menus())
|
|
|
|
| 158 |
PROGRESS_EDIT_EVERY = 5
|
| 159 |
MAX_CPU_PERCENT = max(50, min(95, int(os.environ.get("MAX_CPU_PERCENT", "85"))))
|
| 160 |
CPU_MONITOR_INTERVAL = max(0.5, float(os.environ.get("CPU_MONITOR_INTERVAL", "2.0")))
|
| 161 |
+
COMPRESS_STALL_SECONDS = max(60, int(os.environ.get("COMPRESS_STALL_SECONDS", "180")))
|
|
|
|
|
|
|
|
|
|
| 162 |
QUEUE_WATCHDOG_INTERVAL = max(2.0, float(os.environ.get("QUEUE_WATCHDOG_INTERVAL", "5.0")))
|
| 163 |
CPU_LIMIT_HEADROOM_PERCENT = max(1, min(15, int(os.environ.get("CPU_LIMIT_HEADROOM_PERCENT", "4"))))
|
| 164 |
|
|
|
|
| 980 |
cpu_usage_pct: float = 0.0
|
| 981 |
cpu_slots: int = 0
|
| 982 |
cpu_slots_max: int = 0
|
|
|
|
|
|
|
|
|
|
| 983 |
|
| 984 |
ACTIVE_JOBS: Dict[str, Job] = {}
|
| 985 |
|
|
|
|
| 1122 |
return True
|
| 1123 |
return any(uq.get("processing") == job_id for uq in _user_queues.values())
|
| 1124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1125 |
# =====================
|
| 1126 |
# UTILS
|
| 1127 |
# =====================
|
|
|
|
| 1273 |
job.cpu_slots = current_cpu_slots
|
| 1274 |
job.cpu_slots_max = max_parallel_slots
|
| 1275 |
job.cpu_usage_pct = 0.0
|
|
|
|
|
|
|
|
|
|
| 1276 |
_job_touch(job, stage="comprimiendo", progress=0)
|
| 1277 |
|
| 1278 |
out_time_ms = 0
|
| 1279 |
+
last_progress_at = time.time()
|
|
|
|
|
|
|
|
|
|
| 1280 |
stderr_tail = deque(maxlen=80)
|
| 1281 |
+
t0 = time.time()
|
| 1282 |
ffmpeg_cpu_pct_total = 0.0
|
| 1283 |
ffmpeg_cpu_pct_quota = 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1284 |
|
| 1285 |
async def _read_stdout():
|
| 1286 |
+
nonlocal out_time_ms, last_progress_at
|
| 1287 |
async for line in proc.stdout:
|
|
|
|
|
|
|
|
|
|
| 1288 |
s = line.decode("utf-8", errors="replace").strip()
|
| 1289 |
if "=" not in s:
|
| 1290 |
continue
|
|
|
|
| 1295 |
if new_out_time_ms >= out_time_ms:
|
| 1296 |
out_time_ms = new_out_time_ms
|
| 1297 |
last_progress_at = time.time()
|
|
|
|
|
|
|
| 1298 |
except Exception:
|
| 1299 |
pass
|
| 1300 |
elif k == "progress" and v == "end":
|
| 1301 |
last_progress_at = time.time()
|
|
|
|
|
|
|
| 1302 |
break
|
| 1303 |
|
| 1304 |
async def _read_stderr():
|
|
|
|
| 1305 |
while True:
|
| 1306 |
line = await proc.stderr.readline()
|
| 1307 |
if not line:
|
| 1308 |
break
|
| 1309 |
+
stderr_tail.append(line.decode("utf-8", errors="replace").strip())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1310 |
|
| 1311 |
async def _cpu_guard():
|
| 1312 |
+
nonlocal ffmpeg_cpu_pct_total, ffmpeg_cpu_pct_quota, current_cpu_slots, cpu_subset, active_jobs
|
| 1313 |
if proc.pid is None:
|
| 1314 |
return
|
| 1315 |
try:
|
|
|
|
| 1350 |
job.cpu_usage_pct = ffmpeg_cpu_pct_total
|
| 1351 |
job.cpu_slots = current_cpu_slots
|
| 1352 |
job.cpu_slots_max = max_parallel_slots
|
|
|
|
| 1353 |
_job_touch(job, stage="comprimiendo", progress=(job.progress_pct if job else None))
|
| 1354 |
except Exception:
|
| 1355 |
pass
|
|
|
|
| 1364 |
while not wt.done():
|
| 1365 |
if job and job.cancelled:
|
| 1366 |
raise asyncio.CancelledError()
|
| 1367 |
+
if (time.time() - last_progress_at) > COMPRESS_STALL_SECONDS:
|
| 1368 |
+
raise RuntimeError("La compresión quedó bloqueada sin progreso durante demasiado tiempo.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1369 |
await asyncio.sleep(PROGRESS_EDIT_EVERY)
|
| 1370 |
ds = out_time_ms / 1_000_000 if out_time_ms else 0.0
|
| 1371 |
p = (ds / dur) if dur > 0 else 0.0
|
|
|
|
| 1386 |
f"🧠 CPU: `{cpu_slots_txt} cores` | FFmpeg: `{ffmpeg_cpu_pct_total:.1f}%` host / `{ffmpeg_cpu_pct_quota:.1f}%` cuota\n"
|
| 1387 |
f"🛡️ Tope(host): `{MAX_CPU_PERCENT}%` | Pool: `{max_parallel_slots}` | Jobs: `{active_jobs}`\n"
|
| 1388 |
f"🧩 Cuota cgroup: `{quota_txt}` CPU | Capacidad: `{cpu_capacity:.2f}`\n"
|
|
|
|
| 1389 |
f"⏳ ETA: `{_fmt_eta(eta)}`"
|
| 1390 |
)
|
| 1391 |
await wt
|
|
|
|
| 1408 |
|
| 1409 |
if proc.returncode != 0:
|
| 1410 |
err = "\n".join([x for x in stderr_tail if x][-8:]) or "FFmpeg terminó con error."
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1411 |
raise RuntimeError(err)
|
| 1412 |
if not os.path.exists(out_f) or os.path.getsize(out_f) == 0:
|
| 1413 |
raise RuntimeError("No se generó el archivo comprimido.")
|
|
|
|
| 1545 |
# PROCESO PRINCIPAL
|
| 1546 |
# =====================
|
| 1547 |
async def _process(client, message: Message, status_msg: Message = None, job: Job = None):
|
|
|
|
|
|
|
| 1548 |
editor = _Editor(status_msg, job_id=(job.job_id if job else None))
|
| 1549 |
uid = message.from_user.id
|
| 1550 |
cid = message.chat.id
|
|
|
|
| 1706 |
pass
|
| 1707 |
finally:
|
| 1708 |
if job:
|
|
|
|
| 1709 |
await _cleanup_job(job)
|
| 1710 |
else:
|
| 1711 |
for p in (in_f, out_f, thumb):
|
|
|
|
| 3590 |
|
| 3591 |
lines = ["🧰 **Trabajos del bot**", "━━━━━━━━━━━━━━━━━━━━━"]
|
| 3592 |
for idx, job in enumerate(sorted(active, key=lambda j: (j.is_processing, j.started_at or 0), reverse=True)[:20], 1):
|
| 3593 |
+
state = "🟢 Activo" if job.is_processing else "🟡 En espera"
|
| 3594 |
+
queue_label = _plan_queue_label(job.queue_name or "free") if job.queue_name else "🧵 Cola pendiente"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3595 |
lines.append(
|
| 3596 |
f"{idx}. {state} | `{job.user_id}` | `{job.fname or job.job_id}`\n"
|
| 3597 |
+
f" • Etapa: `{job.stage or 'pendiente'}` | Cola: `{queue_label}`\n"
|
| 3598 |
f" • Progreso: `{job.progress_pct:.1f}%` | CPU: `{job.cpu_usage_pct:.1f}%` | Slots: `{job.cpu_slots}/{job.cpu_slots_max}`\n"
|
| 3599 |
f" • Última actividad: `{_fmt_eta(max(0, time.time() - (job.last_activity_at or time.time())))}` atrás | Tiempo: `{_job_elapsed(job)}`"
|
| 3600 |
)
|
|
|
|
| 4179 |
if _global_worker_task and _global_worker_task.done():
|
| 4180 |
logger.warning("Watchdog: reiniciando worker global con %d elemento(s) pendientes.", len(_global_queue_list))
|
| 4181 |
await _ensure_global_worker()
|
|
|
|
| 4182 |
|
| 4183 |
for queue_key, uq in list(_user_queues.items()):
|
| 4184 |
processing_id = uq.get("processing")
|
|
|
|
| 4190 |
if worker and worker.done():
|
| 4191 |
logger.warning("Watchdog: reiniciando worker %s con %d elemento(s) pendientes.", queue_key, len(uq["list"]))
|
| 4192 |
await _ensure_user_worker(queue_key)
|
|
|
|
| 4193 |
|
| 4194 |
now = time.time()
|
|
|
|
| 4195 |
for job in list(ACTIVE_JOBS.values()):
|
| 4196 |
+
if not job.is_processing or job.cancelled:
|
|
|
|
| 4197 |
continue
|
| 4198 |
+
if job.process and job.process.returncode is None:
|
| 4199 |
+
idle_for = now - (job.last_activity_at or now)
|
| 4200 |
+
if idle_for > (COMPRESS_STALL_SECONDS + 30):
|
| 4201 |
+
logger.warning("Watchdog: trabajo %s cancelado por inactividad (%ss).", job.job_id, int(idle_for))
|
| 4202 |
+
job.cancelled = True
|
| 4203 |
+
await _terminate_process(job.process)
|
| 4204 |
+
elif job.stage == "comprimiendo" and (now - (job.last_activity_at or now)) > (COMPRESS_STALL_SECONDS + 30):
|
| 4205 |
+
logger.warning("Watchdog: trabajo %s quedó sin proceso activo en compresión; se marca cancelado.", job.job_id)
|
| 4206 |
+
job.cancelled = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4207 |
except Exception as e:
|
| 4208 |
logger.error("Queue watchdog error: %s", e)
|
| 4209 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4210 |
# ========================
|
| 4211 |
# COMANDOS DE MENÚ
|
| 4212 |
# ========================
|
|
|
|
| 4222 |
("cancelar", "❌ Cancelar tu compresión activa"),
|
| 4223 |
("reporte", "📝 Reportar un problema"),
|
| 4224 |
("ping", "🏓 Verificar si el bot responde"),
|
|
|
|
|
|
|
| 4225 |
("velocidad", "⚡ Test de velocidad del servidor"),
|
| 4226 |
("id", "🆔 Ver tu ID"),
|
| 4227 |
("about", "ℹ️ Información del bot"),
|
|
|
|
| 4268 |
("importdb", "📥 Importar backup DB"),
|
| 4269 |
("logs", "📡 Configurar canal de logs"),
|
| 4270 |
("forzarwatchdog", "🛠️ Reparar workers/colas atascadas"),
|
|
|
|
| 4271 |
("actualizarmenu", "🔄 Actualizar menú de comandos"),
|
| 4272 |
]
|
| 4273 |
|
|
|
|
| 4342 |
f"🔒 Modo: {BOT_MODE} | 🎬 Log videos: {LOG_VIDEOS}\n"
|
| 4343 |
f"🧵 Colas por plan: {INDIVIDUAL_QUEUES_ENABLED}"
|
| 4344 |
)
|
|
|
|
| 4345 |
asyncio.create_task(_plan_expiry_checker())
|
| 4346 |
asyncio.create_task(_queue_watchdog())
|
| 4347 |
asyncio.create_task(_set_menus())
|