Download bot/task_queue.py from luci18/animebot: direct link, hf CLI and curl.
- Browser
- Download file 12.5 kB
-
https://huggingface.co/luci18/animebot/resolve/main/bot/task_queue.py
- Command line
-
hf download hf://luci18/animebot/bot/task_queue.py
-
curl -L -o task_queue.py https://huggingface.co/luci18/animebot/resolve/main/bot/task_queue.py
12.5 kB
| """Task queue management for downloads and encodings.""" | |
| import asyncio | |
| import logging | |
| import uuid | |
| from typing import Dict, Any, Callable, Optional | |
| from config import Config | |
| from bot.database import db | |
| from bot.downloader import downloader | |
| from bot.encoder import encoder | |
| logger = logging.getLogger(__name__) | |
| class TaskQueue: | |
| """Manages download and encoding queues with concurrency limits.""" | |
| def __init__(self): | |
| self.download_queue: asyncio.Queue = asyncio.Queue() | |
| self.encode_queue: asyncio.Queue = asyncio.Queue() | |
| self.download_semaphore = asyncio.Semaphore(Config.MAX_CONCURRENT_DOWNLOADS) | |
| self.encode_semaphore = asyncio.Semaphore(Config.MAX_CONCURRENT_ENCODINGS) | |
| self.running = False | |
| self.tasks: list = [] | |
| self._progress_callbacks: Dict[str, Callable] = {} | |
| def register_progress_callback(self, task_id: str, callback: Callable): | |
| """Register a progress callback for a task.""" | |
| self._progress_callbacks[task_id] = callback | |
| def unregister_progress_callback(self, task_id: str): | |
| """Unregister a progress callback.""" | |
| self._progress_callbacks.pop(task_id, None) | |
| async def start(self): | |
| """Start the queue workers.""" | |
| self.running = True | |
| # Start download workers | |
| for i in range(Config.MAX_CONCURRENT_DOWNLOADS): | |
| task = asyncio.create_task(self._download_worker()) | |
| self.tasks.append(task) | |
| # Start encode workers | |
| for i in range(Config.MAX_CONCURRENT_ENCODINGS): | |
| task = asyncio.create_task(self._encode_worker()) | |
| self.tasks.append(task) | |
| # Start queue monitor | |
| monitor_task = asyncio.create_task(self._monitor_queue()) | |
| self.tasks.append(monitor_task) | |
| logger.info("Task queue started") | |
| async def stop(self): | |
| """Stop the queue workers.""" | |
| self.running = False | |
| # Cancel all active downloads and encodes | |
| for task_id in list(downloader.active_downloads.keys()): | |
| downloader.cancel_download(task_id) | |
| for task_id in list(encoder.active_encodes.keys()): | |
| encoder.cancel_encode(task_id) | |
| # Cancel workers | |
| for task in self.tasks: | |
| task.cancel() | |
| await asyncio.gather(*self.tasks, return_exceptions=True) | |
| logger.info("Task queue stopped") | |
| async def _monitor_queue(self): | |
| """Monitor queue and add pending tasks from database.""" | |
| while self.running: | |
| try: | |
| # Check for pending downloads in database | |
| pending_downloads = db.get_pending_downloads() | |
| for dl in pending_downloads: | |
| await self.add_download( | |
| task_id=dl["task_id"], | |
| anime_url=dl["anime_id"], | |
| episode_num=dl.get("episode_num", 1), | |
| user_id=dl["user_id"] | |
| ) | |
| # Check for pending encodes in database | |
| pending_encodes = db.get_pending_encodes() | |
| for enc in pending_encodes: | |
| await self.add_encode( | |
| task_id=enc["task_id"], | |
| input_path=enc["input_path"], | |
| user_id=enc["user_id"], | |
| settings=enc.get("settings", {}) | |
| ) | |
| await asyncio.sleep(5) # Check every 5 seconds | |
| except Exception as e: | |
| logger.error(f"Queue monitor error: {e}") | |
| await asyncio.sleep(10) | |
| async def add_download(self, task_id: str, anime_url: str, | |
| episode_num: int, user_id: int, | |
| site: Optional[str] = None) -> str: | |
| """Add a download task to the queue.""" | |
| task = { | |
| "task_id": task_id, | |
| "type": "download", | |
| "anime_url": anime_url, | |
| "episode_num": episode_num, | |
| "user_id": user_id, | |
| "site": site | |
| } | |
| await self.download_queue.put(task) | |
| logger.info(f"Added download task {task_id} to queue (site: {site})") | |
| return task_id | |
| async def add_encode(self, task_id: str, input_path: str, | |
| user_id: int, settings: Dict[str, Any]) -> str: | |
| """Add an encode task to the queue.""" | |
| task = { | |
| "task_id": task_id, | |
| "type": "encode", | |
| "input_path": input_path, | |
| "user_id": user_id, | |
| "settings": settings | |
| } | |
| await self.encode_queue.put(task) | |
| logger.info(f"Added encode task {task_id} to queue") | |
| return task_id | |
| async def _download_worker(self): | |
| """Worker to process download tasks.""" | |
| while self.running: | |
| try: | |
| task = await asyncio.wait_for(self.download_queue.get(), timeout=1.0) | |
| except asyncio.TimeoutError: | |
| continue | |
| async with self.download_semaphore: | |
| await self._process_download(task) | |
| self.download_queue.task_done() | |
| async def _encode_worker(self): | |
| """Worker to process encode tasks.""" | |
| while self.running: | |
| try: | |
| task = await asyncio.wait_for(self.encode_queue.get(), timeout=1.0) | |
| except asyncio.TimeoutError: | |
| continue | |
| async with self.encode_semaphore: | |
| await self._process_encode(task) | |
| self.encode_queue.task_done() | |
| async def _process_download(self, task: Dict): | |
| """Process a single download task.""" | |
| task_id = task["task_id"] | |
| anime_url = task["anime_url"] | |
| episode_num = task["episode_num"] | |
| user_id = task["user_id"] | |
| site = task.get("site") | |
| progress_callback = self._progress_callbacks.get(task_id) | |
| def sync_download(): | |
| def wrapped_progress(progress: int, message: str): | |
| if progress_callback: | |
| # Schedule callback in event loop | |
| asyncio.create_task(self._safe_callback(progress_callback, progress, message)) | |
| return downloader.download_episode( | |
| anime_url=anime_url, | |
| episode_num=episode_num, | |
| user_id=user_id, | |
| site=site, | |
| progress_callback=wrapped_progress if progress_callback else None | |
| ) | |
| try: | |
| loop = asyncio.get_event_loop() | |
| result = await loop.run_in_executor(None, sync_download) | |
| if result: | |
| # Apply renaming if enabled | |
| settings = db.get_user_settings(user_id) | |
| if settings.get("auto_rename", True): | |
| try: | |
| from bot.renamer import renamer | |
| from pathlib import Path | |
| # Get anime info for metadata | |
| anime_info = await loop.run_in_executor( | |
| None, lambda: downloader.get_anime_info(anime_url, site) | |
| ) | |
| if anime_info: | |
| anime_info["source_site"] = site or "" | |
| # Extract metadata | |
| metadata = renamer.extract_metadata( | |
| Path(result), | |
| anime_info=anime_info, | |
| episode_num=episode_num | |
| ) | |
| # Rename file | |
| pattern = settings.get("rename_pattern", Config.DEFAULT_RENAME_PATTERN) | |
| new_path = renamer.rename_file( | |
| Path(result), | |
| pattern, | |
| metadata | |
| ) | |
| if new_path: | |
| result = new_path | |
| logger.info(f"Renamed download to: {new_path.name}") | |
| except Exception as e: | |
| logger.warning(f"Rename failed for download: {e}") | |
| if task.get("auto_encode"): | |
| # Auto-encode after download | |
| await self.add_encode( | |
| task_id=str(uuid.uuid4()), | |
| input_path=str(result), | |
| user_id=user_id, | |
| settings=settings | |
| ) | |
| except Exception as e: | |
| logger.error(f"Download task {task_id} failed: {e}") | |
| finally: | |
| self.unregister_progress_callback(task_id) | |
| async def _process_encode(self, task: Dict): | |
| """Process a single encode task.""" | |
| from pathlib import Path | |
| task_id = task["task_id"] | |
| input_path = Path(task["input_path"]) | |
| user_id = task["user_id"] | |
| settings = task["settings"] | |
| progress_callback = self._progress_callbacks.get(task_id) | |
| def sync_encode(): | |
| def wrapped_progress(progress: int, message: str): | |
| if progress_callback: | |
| asyncio.create_task(self._safe_callback(progress_callback, progress, message)) | |
| return encoder.encode_video( | |
| input_path=input_path, | |
| user_id=user_id, | |
| settings=settings, | |
| progress_callback=wrapped_progress if progress_callback else None | |
| ) | |
| try: | |
| loop = asyncio.get_event_loop() | |
| result = await loop.run_in_executor(None, sync_encode) | |
| if result: | |
| logger.info(f"Encode task {task_id} completed: {result}") | |
| # Apply renaming if enabled | |
| if settings.get("auto_rename", True): | |
| try: | |
| from bot.renamer import renamer | |
| # Extract metadata from encoded file | |
| metadata = renamer.extract_metadata(Path(result)) | |
| # Use encode-specific pattern or default | |
| pattern = settings.get("rename_pattern", Config.DEFAULT_RENAME_PATTERN) | |
| # Add [ENCODED] tag for encoded files | |
| pattern = pattern.replace("{title}", "{title} [ENCODED]") | |
| new_path = renamer.rename_file( | |
| Path(result), | |
| pattern, | |
| metadata | |
| ) | |
| if new_path: | |
| result = new_path | |
| logger.info(f"Renamed encode to: {new_path.name}") | |
| except Exception as e: | |
| logger.warning(f"Rename failed for encode: {e}") | |
| else: | |
| logger.error(f"Encode task {task_id} failed") | |
| except Exception as e: | |
| logger.error(f"Encode task {task_id} failed: {e}") | |
| finally: | |
| self.unregister_progress_callback(task_id) | |
| async def _safe_callback(self, callback: Callable, *args, **kwargs): | |
| """Safely invoke a callback, catching exceptions.""" | |
| try: | |
| if asyncio.iscoroutinefunction(callback): | |
| await callback(*args, **kwargs) | |
| else: | |
| callback(*args, **kwargs) | |
| except Exception as e: | |
| logger.error(f"Callback error: {e}") | |
| def get_queue_status(self) -> Dict[str, Any]: | |
| """Get current queue status.""" | |
| active = db.get_active_tasks() | |
| return { | |
| "download_queue_size": self.download_queue.qsize(), | |
| "encode_queue_size": self.encode_queue.qsize(), | |
| "active_downloads": active["downloads"], | |
| "active_encodes": active["encodes"], | |
| "max_downloads": Config.MAX_CONCURRENT_DOWNLOADS, | |
| "max_encodes": Config.MAX_CONCURRENT_ENCODINGS | |
| } | |
| # Global queue instance | |
| task_queue = TaskQueue() | |