""" AudioNovel Telegram Bot ======================== Hosted on Hugging Face Spaces (CPU-free tier). Send a .txt / .epub / .pdf to the bot → get MP3 + MP4 audiobook back, files are also pushed to your HF bucket automatically. Environment variables required (set in HF Space → Settings → Variables & Secrets): BOT_TOKEN – Telegram bot token from @BotFather HF_TOKEN – Hugging Face write token (Settings → Access Tokens) HF_BUCKET_REPO – e.g. Krixter/storage """ import os, asyncio, tempfile, logging, shutil, time, re, zipfile, subprocess, requests from concurrent.futures import ThreadPoolExecutor, as_completed from bs4 import BeautifulSoup from huggingface_hub import HfApi from telegram import Update, BotCommand from telegram.ext import ( ApplicationBuilder, MessageHandler, CommandHandler, filters, ContextTypes ) from telegram.request import HTTPXRequest # ─── CONFIG ──────────────────────────────────────────────────────────────────── BOT_TOKEN = os.environ.get("BOT_TOKEN", "") HF_TOKEN = os.environ.get("HF_TOKEN", "") HF_BUCKET_REPO = os.environ.get("HF_BUCKET_REPO", "Krixter/storage") HF_REPO_TYPE = "dataset" # HF buckets are dataset repos FIXED_SPEED = 1.6 THREADS = 6 # lower than original – HF free CPU has 2 vCPUs MAX_TG_FILE_MB = 20 # Telegram Bot API hard limit for received files logging.basicConfig( format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", level=logging.INFO ) log = logging.getLogger("AudioBot") # ─── FAKE PROGRESS (no gr.Progress on Telegram) ──────────────────────────────── class TelegramProgress: """Sends throttled progress updates back to the Telegram chat.""" def __init__(self, update: Update, loop): self.update = update self.loop = loop self._last = 0.0 self._msg_id = None def __call__(self, value: float, desc: str = ""): # Only update every ~5 % to avoid Telegram rate-limits if value - self._last < 0.05 and value < 0.99: return self._last = value text = f"⏳ {int(value * 100)}% {desc}" asyncio.run_coroutine_threadsafe(self._send(text), self.loop) async def _send(self, text: str): try: if self._msg_id: await self.update.message.bot.edit_message_text( chat_id=self.update.effective_chat.id, message_id=self._msg_id, text=text ) else: sent = await self.update.message.reply_text(text) self._msg_id = sent.message_id except Exception: pass # edit conflicts are harmless # ─── TEXT EXTRACTION ─────────────────────────────────────────────────────────── def extract_local_text(path: str, temp_dir: str) -> str | None: content = "" ext = os.path.splitext(path)[1].lower() try: if ext == ".txt": with open(path, "r", encoding="utf-8", errors="ignore") as f: content = f.read() elif ext == ".epub": with zipfile.ZipFile(path, "r") as z: try: container = z.read("META-INF/container.xml") soup = BeautifulSoup(container, "html.parser") opf_path = soup.find("rootfile").get("full-path") opf_dir = os.path.dirname(opf_path) opf_soup = BeautifulSoup(z.read(opf_path), "html.parser") manifest = {item.get("id"): item.get("href") for item in opf_soup.find_all("item")} for itemref in opf_soup.find_all("itemref"): item_id = itemref.get("idref") if item_id not in manifest: continue href = manifest[item_id] file_path = (href if not opf_dir else f"{opf_dir}/{href}").split("#")[0] try: html_soup = BeautifulSoup(z.read(file_path), "html.parser") clean_part = html_soup.get_text(separator="\n") if clean_part: content += clean_part + "\n" except KeyError: continue except Exception as e: log.warning(f"EPUB spine parse failed, using fallback: {e}") for filename in z.namelist(): if filename.endswith((".html", ".xhtml")): with z.open(filename) as f: fb = BeautifulSoup(f.read(), "html.parser") part = fb.get_text(separator="\n") if part: content += part + "\n" elif ext == ".pdf": try: import PyPDF2 reader = PyPDF2.PdfReader(path) for page in reader.pages: txt = page.extract_text() if txt: content += txt + "\n" except ImportError: log.error("PyPDF2 not installed – PDF support unavailable") return None except Exception as e: log.error(f"Text extraction failed: {e}") return None if not content.strip(): return None content = re.sub(r"[ \t]+", " ", content) content = re.sub(r"\n\s*\n", "\n\n", content) out = os.path.join(temp_dir, "extracted.txt") with open(out, "w", encoding="utf-8") as f: f.write(content) return out # ─── TTS ENGINE ──────────────────────────────────────────────────────────────── class GoogleTTSEngine: def split_text(self, text: str, max_len: int = 200) -> list[str]: chunks = [] while text: if len(text) <= max_len: chunks.append(text.strip()) break best_i = -1 for punct in [". ", "! ", "? ", "; "]: i = text[:max_len].rfind(punct) if i > best_i: best_i = i if best_i == -1: best_i = text[:max_len].rfind(" ") if best_i == -1: best_i = max_len else: best_i += 1 chunks.append(text[:best_i].strip()) text = text[best_i:].lstrip() return chunks @staticmethod def strip_id3(mp3_data: bytes) -> bytes: if len(mp3_data) >= 10 and mp3_data[:3] == b"ID3": size = ((mp3_data[6] & 0x7F) << 21 | (mp3_data[7] & 0x7F) << 14 | (mp3_data[8] & 0x7F) << 7 | (mp3_data[9] & 0x7F)) mp3_data = mp3_data[10 + size:] if len(mp3_data) > 128 and mp3_data[-128:-125] == b"TAG": mp3_data = mp3_data[:-128] return mp3_data def download_chunk(self, text: str) -> bytes | None: url = "https://translate.google.com/translate_tts" params = {"ie": "UTF-8", "q": text, "tl": "en-IN", "client": "tw-ob"} headers = {"User-Agent": "Mozilla/5.0"} for attempt in range(6): try: r = requests.get(url, params=params, headers=headers, timeout=8) if r.status_code == 200: return self.strip_id3(r.content) # 429 = rate-limit → back off if r.status_code == 429: time.sleep(2 ** attempt) except requests.RequestException: time.sleep(0.5) return None # ─── CORE PIPELINE ───────────────────────────────────────────────────────────── def sync_process_book(file_path: str, temp_dir: str, progress, book_idx: int, total_books: int): text_file = extract_local_text(file_path, temp_dir) if not text_file: return None, None, None base_name = os.path.splitext(os.path.basename(file_path))[0] book_label = f"[Book {book_idx}/{total_books}]" with open(text_file, "r", encoding="utf-8") as f: full_text = f.read() # ── Chapter Detection ────────────────────────────────────────────────────── splits = re.split( r"(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*|\b\d+\s*:(?!\d)[^\n]*)", full_text ) chapters = [] if splits[0].strip(): chapters.append(["Intro", splits[0].replace("\n", " ").strip()]) for i in range(1, len(splits), 2): title = splits[i].strip() body = splits[i + 1].strip() if i + 1 < len(splits) else "" if len(title) > 100: body = title[100:] + " " + body title = title[:100] + "..." num_match = re.search(r"\d+", title) current_num = num_match.group() if num_match else title if chapters: prev_title = chapters[-1][0] prev_num_match = re.search(r"\d+", prev_title) prev_num = prev_num_match.group() if prev_num_match else prev_title if current_num == prev_num: # double heading chapters[-1][1] += f" {body}".replace("\n", " ") continue if len(chapters[-1][1]) < 150: # table-of-contents / tiny chapter chapters[-1][1] += f" {title}. {body}".replace("\n", " ") chapters[-1][0] = title continue clean_text_for_tts = f"{title}. {body}".replace("\n", " ") chapters.append([title, clean_text_for_tts]) if not chapters: chapters = [["Full Book", full_text.replace("\n", " ")]] engine = GoogleTTSEngine() all_chunks = [] for c_idx, (title, c_text) in enumerate(chapters): for chunk in engine.split_text(c_text): if chunk.strip(): all_chunks.append((c_idx, chunk)) total_chunks = len(all_chunks) chunk_dir = os.path.join(temp_dir, "chunks") os.makedirs(chunk_dir, exist_ok=True) # ── Download TTS chunks ──────────────────────────────────────────────────── start_time = time.time() completed = 0 with ThreadPoolExecutor(max_workers=THREADS) as pool: futures = {} c_chunk_counters = {c_idx: 0 for c_idx in range(len(chapters))} for c_idx, chunk in all_chunks: fut = pool.submit(engine.download_chunk, chunk) futures[fut] = (c_idx, c_chunk_counters[c_idx]) c_chunk_counters[c_idx] += 1 for fut in as_completed(futures): c_idx, local_c_idx = futures[fut] data = fut.result() if data: out_path = os.path.join(chunk_dir, f"c{c_idx:03d}_{local_c_idx:05d}.mp3") with open(out_path, "wb") as af: af.write(data) completed += 1 elapsed = time.time() - start_time avg = elapsed / completed eta = avg * (total_chunks - completed) progress( completed / total_chunks, desc=f"📥 {book_label} {completed}/{total_chunks} | ETA {int(eta//60):02d}:{int(eta%60):02d}" ) # ── Merge & Speed-up per chapter ────────────────────────────────────────── processed_chapters = [] metadata_entries = [] current_time_ms = 0 for c_idx, (title, _) in enumerate(chapters): c_files = sorted( f for f in os.listdir(chunk_dir) if f.startswith(f"c{c_idx:03d}_") ) if not c_files: continue raw_c_path = os.path.join(temp_dir, f"raw_c{c_idx}.mp3") with open(raw_c_path, "wb") as outfile: for cf in c_files: with open(os.path.join(chunk_dir, cf), "rb") as infile: shutil.copyfileobj(infile, outfile) proc_c_path = os.path.join(temp_dir, f"proc_c{c_idx}.mp3") result = subprocess.run( ["ffmpeg", "-y", "-i", raw_c_path, "-af", f"silenceremove=1:0:-50dB,atempo={FIXED_SPEED}", "-threads", "0", proc_c_path], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE ) if result.returncode != 0: log.warning(f"ffmpeg chapter {c_idx} failed: {result.stderr.decode()[:200]}") continue # skip broken chapter rather than crashing processed_chapters.append(proc_c_path) dur_result = subprocess.run( ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", proc_c_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) try: duration_ms = int(float(dur_result.stdout.decode().strip()) * 1000) except (ValueError, AttributeError): duration_ms = 0 metadata_entries.append({"title": title, "start": current_time_ms, "end": current_time_ms + duration_ms}) current_time_ms += duration_ms return processed_chapters, metadata_entries, base_name def final_merge(processed_chapters: list, metadata_entries: list, base_name: str, temp_dir: str) -> list[str]: if not processed_chapters: return [] # ── FFMETADATA ──────────────────────────────────────────────────────────── meta_path = os.path.join(temp_dir, "metadata.txt") with open(meta_path, "w", encoding="utf-8") as f: f.write(";FFMETADATA1\n") f.write(f"title={base_name}\n\n") for entry in metadata_entries: f.write("[CHAPTER]\nTIMEBASE=1/1000\n") f.write(f"START={entry['start']}\nEND={entry['end']}\ntitle={entry['title']}\n\n") # ── Concat list ─────────────────────────────────────────────────────────── concat_path = os.path.join(temp_dir, "concat.txt") with open(concat_path, "w", encoding="utf-8") as f: for p in processed_chapters: f.write(f"file '{os.path.abspath(p).replace(chr(92), '/')}'\n") out_dir = tempfile.gettempdir() out_mp3 = os.path.join(out_dir, f"{base_name}_Audio.mp3") out_mp4 = os.path.join(out_dir, f"{base_name}_Audio.mp4") # MP3 – stream copy (fast) r1 = subprocess.run( ["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path, "-i", meta_path, "-map_metadata", "1", "-c", "copy", out_mp3], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE ) if r1.returncode != 0: log.error(f"MP3 merge failed: {r1.stderr.decode()[:300]}") # MP4 – AAC encode for chapter-marker compatibility r2 = subprocess.run( ["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path, "-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", out_mp4], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE ) if r2.returncode != 0: log.error(f"MP4 merge failed: {r2.stderr.decode()[:300]}") shutil.rmtree(temp_dir, ignore_errors=True) return [f for f in [out_mp3, out_mp4] if os.path.isfile(f)] # ─── HF UPLOAD ───────────────────────────────────────────────────────────────── def upload_to_hf(file_paths: list[str]) -> list[str]: """Upload files to HF dataset repo and return public download URLs.""" api = HfApi(token=HF_TOKEN) links = [] for fpath in file_paths: if not os.path.isfile(fpath): continue fname = os.path.basename(fpath) try: api.upload_file( path_or_fileobj=fpath, path_in_repo=fname, repo_id=HF_BUCKET_REPO, repo_type=HF_REPO_TYPE, ) link = (f"https://huggingface.co/datasets/" f"{HF_BUCKET_REPO}/resolve/main/{fname}") links.append(link) log.info(f"✅ Uploaded {fname} → {link}") except Exception as e: log.error(f"HF upload failed for {fname}: {e}") return links # ─── LIST FILES IN BUCKET ────────────────────────────────────────────────────── def list_hf_files() -> list[str]: try: api = HfApi(token=HF_TOKEN) files = api.list_repo_files(repo_id=HF_BUCKET_REPO, repo_type=HF_REPO_TYPE) return list(files) except Exception as e: log.error(f"list_hf_files failed: {e}") return [] # ─── TELEGRAM HANDLERS ───────────────────────────────────────────────────────── async def cmd_start(update: Update, context: ContextTypes.DEFAULT_TYPE): await update.message.reply_text( "👋 *AudioNovel Bot*\n\n" "Send me a `.txt`, `.epub`, or `.pdf` file and I'll convert it to an audiobook " "(MP3 + MP4) and upload it to your HF bucket.\n\n" "📂 /list — show files already in the bucket\n" "ℹ️ /help — usage tips", parse_mode="Markdown" ) async def cmd_help(update: Update, context: ContextTypes.DEFAULT_TYPE): await update.message.reply_text( "📖 *How to use*\n\n" "1. Send any `.txt`, `.epub`, or `.pdf` file (max 20 MB)\n" "2. Wait — conversion takes a few minutes per book\n" "3. I'll reply with download links from your HF bucket\n\n" "📂 /list — browse existing files in the bucket", parse_mode="Markdown" ) async def cmd_list(update: Update, context: ContextTypes.DEFAULT_TYPE): await update.message.reply_text("🔍 Fetching file list from HF bucket…") files = await asyncio.to_thread(list_hf_files) if not files: await update.message.reply_text("📭 No files found in the bucket yet.") return base_url = f"https://huggingface.co/datasets/{HF_BUCKET_REPO}/resolve/main/" lines = [f"📄 [{f}]({base_url}{f})" for f in files] # Telegram has a 4096-char message limit; chunk if needed chunk, chunks = "", [] for line in lines: if len(chunk) + len(line) + 1 > 4000: chunks.append(chunk) chunk = "" chunk += line + "\n" if chunk: chunks.append(chunk) for part in chunks: await update.message.reply_text(part, parse_mode="Markdown", disable_web_page_preview=True) async def handle_file(update: Update, context: ContextTypes.DEFAULT_TYPE): doc = update.message.document if not doc: await update.message.reply_text("Please send a book file (.txt / .epub / .pdf).") return fname = doc.file_name or "book" ext = os.path.splitext(fname)[1].lower() if ext not in (".txt", ".epub", ".pdf"): await update.message.reply_text( "❌ Unsupported format.\n" "Please send a `.txt`, `.epub`, or `.pdf` file." ) return size_mb = (doc.file_size or 0) / (1024 * 1024) if size_mb > MAX_TG_FILE_MB: await update.message.reply_text( f"❌ File too large ({size_mb:.1f} MB).\n" f"Telegram's limit for bots is {MAX_TG_FILE_MB} MB.\n" "Try splitting the book into smaller parts." ) return await update.message.reply_text( f"📥 Received *{fname}* ({size_mb:.1f} MB)\n" "Starting conversion… this may take several minutes ☕", parse_mode="Markdown" ) loop = asyncio.get_event_loop() progress = TelegramProgress(update, loop) temp_dir = tempfile.mkdtemp() try: # 1. Download file from Telegram tg_file = await doc.get_file() local_path = os.path.join(temp_dir, fname) await tg_file.download_to_drive(local_path) # 2. Run TTS pipeline in a thread (blocking) processed_chapters, metadata_entries, base_name = await asyncio.to_thread( sync_process_book, local_path, temp_dir, progress, 1, 1 ) if not processed_chapters: await update.message.reply_text( "❌ Could not extract readable text from the file.\n" "Make sure it is not password-protected or corrupted." ) return # 3. Merge chapters → MP3 + MP4 await update.message.reply_text("🔀 Merging chapters…") output_files = await asyncio.to_thread( final_merge, processed_chapters, metadata_entries, base_name, temp_dir ) if not output_files: await update.message.reply_text("❌ Merge failed. Check Space logs for details.") return # 4. Upload to HF bucket await update.message.reply_text("☁️ Uploading to HF bucket…") links = await asyncio.to_thread(upload_to_hf, output_files) if links: msg = "✅ *Done! Your audiobook is ready:*\n\n" for link in links: msg += f"🔗 {link}\n" await update.message.reply_text(msg, parse_mode="Markdown", disable_web_page_preview=False) else: await update.message.reply_text( "⚠️ Conversion succeeded but HF upload failed.\n" "Check that your HF_TOKEN has *write* permission.", parse_mode="Markdown" ) except Exception as e: log.exception("handle_file crashed") await update.message.reply_text(f"❌ Unexpected error: {e}") finally: # Clean up temp dir if final_merge hasn't already done it shutil.rmtree(temp_dir, ignore_errors=True) # ─── MAIN ────────────────────────────────────────────────────────────────────── def main(): if not BOT_TOKEN: raise RuntimeError("BOT_TOKEN environment variable is not set!") if not HF_TOKEN: log.warning("HF_TOKEN not set – uploads will likely fail.") request = HTTPXRequest( connect_timeout=30.0, read_timeout=30.0, write_timeout=30.0 ) app = ApplicationBuilder().token(BOT_TOKEN).request(request).build() app.add_handler(CommandHandler("start", cmd_start)) app.add_handler(CommandHandler("help", cmd_help)) app.add_handler(CommandHandler("list", cmd_list)) app.add_handler(MessageHandler(filters.Document.ALL, handle_file)) log.info("🤖 AudioNovel bot is running…") app.run_polling(drop_pending_updates=True) if __name__ == "__main__": main()