Spaces:
Sleeping
Sleeping
Download app.py from Krixter/Ultimate_tts: direct link, hf CLI and curl.
- Browser
- Download file 35.4 kB
-
https://huggingface.co/spaces/Krixter/Ultimate_tts/resolve/main/app.py
- Command line
-
hf download hf://spaces/Krixter/Ultimate_tts/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Krixter/Ultimate_tts/resolve/main/app.py
35.4 kB
| import gradio as gr | |
| import os | |
| import tempfile | |
| import requests | |
| import subprocess | |
| import zipfile | |
| import shutil | |
| import time | |
| import asyncio | |
| import re | |
| from bs4 import BeautifulSoup | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from huggingface_hub import HfFileSystem | |
| # --- CONFIGURATION --- | |
| FIXED_SPEED = 1.6 | |
| THREADS = 10 | |
| # --- HF BUCKET --- | |
| HF_BUCKET = "hf://buckets/Krixter/storage" | |
| HF_LOCAL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") | |
| def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool: | |
| """Upload finished audio files directly to the HF bucket via HfFileSystem.""" | |
| token = os.environ.get("HF_TOKEN") | |
| if not token: | |
| print("β οΈ HF_TOKEN secret not set β bucket upload skipped.") | |
| return False | |
| try: | |
| fs = HfFileSystem(token=token) | |
| uploaded = 0 | |
| for src in files: | |
| if not src or not os.path.isfile(src): | |
| continue | |
| dest = f"{HF_BUCKET}/{os.path.basename(src)}" | |
| print(f"π€ {book_label} Uploading {os.path.basename(src)} β {dest}") | |
| fs.put_file(src, dest) | |
| print(f"βοΈ {book_label} Uploaded OK: {os.path.basename(src)}") | |
| uploaded += 1 | |
| return uploaded > 0 | |
| except Exception as e: | |
| print(f"β οΈ {book_label} Bucket upload failed: {e}") | |
| return False | |
| # --- THE BOUNCERS --- | |
| download_door = asyncio.Semaphore(1) | |
| kitchen_door = asyncio.Semaphore(1) | |
| def extract_local_text(path, temp_dir): | |
| """ | |
| Returns (text_file_path, epub_chapters_or_None). | |
| epub_chapters is a list of (title, body_text) pairs when the file is an EPUB | |
| and heading-based chapter extraction succeeded. In all other cases it is None, | |
| and the caller should use detect_and_split() on the text file. | |
| """ | |
| content = "" | |
| epub_chapters = None # β NEW: populated only for EPUBs | |
| ext = os.path.splitext(path)[1].lower() | |
| try: | |
| if ext == '.txt': | |
| with open(path, 'r', encoding='utf-8', errors='ignore') as f: | |
| content = f.read() | |
| # ββ BULLETPROOF EPUB BLOCK ββββββββββββββββββββββββββββββββββββββββββββ | |
| elif ext == '.epub': | |
| with zipfile.ZipFile(path, 'r') as z: | |
| try: | |
| # 1. Find the master OPF file | |
| container = z.read('META-INF/container.xml') | |
| c_soup = BeautifulSoup(container, 'html.parser') | |
| opf_path = c_soup.find('rootfile').get('full-path') | |
| opf_dir = os.path.dirname(opf_path) | |
| opf_content = z.read(opf_path) | |
| opf_soup = BeautifulSoup(opf_content, 'html.parser') | |
| # 2. Build the Manifest | |
| manifest = {} | |
| for item in opf_soup.find_all('item'): | |
| manifest[item.get('id')] = item.get('href') | |
| # 3. Try heading-based chapter extraction (NEW) | |
| epub_chapters = _epub_extract_chapters_from_spine(z, opf_soup, opf_dir, manifest) | |
| # 4. Always build the flat text too (needed as fallback) | |
| for itemref in opf_soup.find_all('itemref'): | |
| item_id = itemref.get('idref') | |
| if item_id not in manifest: | |
| continue | |
| href = manifest[item_id] | |
| file_path = href if not opf_dir else f"{opf_dir}/{href}" | |
| file_path = file_path.split('#')[0] | |
| try: | |
| html_data = z.read(file_path) | |
| html_soup = BeautifulSoup(html_data, 'html.parser') | |
| clean_part = html_soup.get_text(separator='\n') | |
| if clean_part: | |
| content += clean_part + "\n" | |
| except KeyError: | |
| continue | |
| except Exception as e: | |
| print(f"β οΈ Spine/heading parse failed, using fallback mode: {e}") | |
| epub_chapters = None | |
| for filename in z.namelist(): | |
| if filename.endswith(('.html', '.xhtml')): | |
| with z.open(filename) as f: | |
| sb = BeautifulSoup(f.read(), 'html.parser') | |
| cp = sb.get_text(separator='\n') | |
| if cp: | |
| content += cp + "\n" | |
| # ββ END EPUB BLOCK ββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| elif ext == '.pdf': | |
| import PyPDF2 | |
| reader = PyPDF2.PdfReader(path) | |
| for page in reader.pages: | |
| txt = page.extract_text() | |
| if txt: | |
| content += txt + "\n" | |
| except Exception as e: | |
| return None, None | |
| if not content and epub_chapters is None: | |
| return None, None | |
| # Clean up horizontal whitespace; preserve newlines | |
| content = re.sub(r'[ \t]+', ' ', content) | |
| content = re.sub(r'\n\s*\n', '\n\n', content) | |
| out = os.path.join(temp_dir, "extracted.txt") | |
| with open(out, 'w', encoding='utf-8') as f: | |
| f.write(content) | |
| return out, epub_chapters | |
| class GoogleTTSEngine: | |
| def split_text(self, text, max_len=200): | |
| """SMART SPLIT: Prioritizes cutting at sentence endings to reduce robotic pauses.""" | |
| chunks = [] | |
| while text: | |
| if len(text) <= max_len: | |
| chunks.append(text.strip()) | |
| break | |
| best_i = -1 | |
| for punct in [". ", "! ", "? ", "; "]: | |
| i = text[:max_len].rfind(punct) | |
| if i > best_i: | |
| best_i = i | |
| if best_i == -1: | |
| best_i = text[:max_len].rfind(" ") | |
| if best_i == -1: | |
| best_i = max_len | |
| else: | |
| best_i += 1 | |
| chunks.append(text[:best_i].strip()) | |
| text = text[best_i:].lstrip() | |
| return chunks | |
| def strip_id3(self, mp3_data): | |
| if len(mp3_data) >= 10 and mp3_data[:3] == b"ID3": | |
| size = ((mp3_data[6] & 0x7F) << 21 | (mp3_data[7] & 0x7F) << 14 | | |
| (mp3_data[8] & 0x7F) << 7 | (mp3_data[9] & 0x7F)) | |
| mp3_data = mp3_data[10 + size:] | |
| if len(mp3_data) > 128 and mp3_data[-128:-125] == b"TAG": | |
| mp3_data = mp3_data[:-128] | |
| return mp3_data | |
| def download_chunk(self, text): | |
| url = "https://translate.google.com/translate_tts" | |
| params = {"ie": "UTF-8", "q": text, "tl": "en-IN", "client": "tw-ob"} | |
| headers = {"User-Agent": "Mozilla/5.0"} | |
| for _ in range(5): | |
| try: | |
| r = requests.get(url, params=params, headers=headers, timeout=5) | |
| if r.status_code == 200: return self.strip_id3(r.content) | |
| except: time.sleep(0.5) | |
| return None | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # CHAPTER DETECTION ENGINE | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| _WORD_NUMS = ( | |
| "zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|" | |
| "thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|" | |
| "thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred" | |
| ) | |
| # Roman numeral pattern (standalone, not substrings like "VI" inside a word) | |
| _ROMAN_PAT = r"(?:M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3}))" | |
| def _roman_to_int(s): | |
| """Convert a Roman numeral string to an integer (returns 0 on failure).""" | |
| vals = {'I':1,'V':5,'X':10,'L':50,'C':100,'D':500,'M':1000} | |
| s = s.upper().strip() | |
| if not s: | |
| return 0 | |
| result, prev = 0, 0 | |
| for ch in reversed(s): | |
| v = vals.get(ch, 0) | |
| if v < prev: | |
| result -= v | |
| else: | |
| result += v | |
| prev = v | |
| return result | |
| _WORD_NUM_MAP = {w: i for i, w in enumerate(_WORD_NUMS.split("|"))} | |
| def _extract_num(text, word_mode=False, roman_mode=False): | |
| """ | |
| Pull a chapter sequence number out of a heading string. | |
| Tries Arabic β Roman β Word in that order (unless a specific mode is set). | |
| Returns an integer or None. | |
| """ | |
| # Arabic first β always tried | |
| m = re.search(r"\d+", text) | |
| if m: | |
| return int(m.group()) | |
| if roman_mode: | |
| m = re.search(r"\b(" + _ROMAN_PAT + r")\b", text, re.I) | |
| if m and m.group(1).upper() != "I": # lone "I" is too ambiguous | |
| return _roman_to_int(m.group(1)) | |
| if word_mode: | |
| m = re.search(r"\b(" + _WORD_NUMS + r")\b", text, re.I) | |
| if m: | |
| return _WORD_NUM_MAP.get(m.group(1).lower(), -1) | |
| return None | |
| def _is_ascending(candidates, word_mode=False, roman_mode=False): | |
| """ | |
| Return (is_valid, ratio). | |
| Valid when β₯60 % of consecutive pairs are strictly ascending. | |
| Deduplicates runs of the same number first (EPUB double-heading artefact). | |
| Skips un-parseable candidates rather than failing. | |
| Requires at least 2 parseable, distinct numbers. | |
| """ | |
| nums = [] | |
| for c in candidates: | |
| n = _extract_num(c, word_mode=word_mode, roman_mode=roman_mode) | |
| if n is not None: | |
| nums.append(n) | |
| # Deduplicate consecutive equal numbers (e.g. 1,1,2,2,3,3 β 1,2,3) | |
| deduped = [] | |
| for n in nums: | |
| if not deduped or n != deduped[-1]: | |
| deduped.append(n) | |
| if len(deduped) < 2: | |
| return False, 0.0 | |
| ascending = sum(1 for j in range(1, len(deduped)) if deduped[j] > deduped[j-1]) | |
| ratio = ascending / (len(deduped) - 1) | |
| return ratio >= 0.6, ratio | |
| def _title_has_words(heading): | |
| """True if the heading contains non-numeric, non-separator words beyond the keyword.""" | |
| # Strip common chapter-keywords and numbers, see if real words remain | |
| cleaned = re.sub( | |
| r"(?i)\b(?:chapter|ch\.?|part|book|episode|arc|volume|vol\.?|act|section)\b", "", heading | |
| ) | |
| cleaned = re.sub(r"[\d\s\-:./|()]+", " ", cleaned).strip() | |
| return bool(cleaned) | |
| # ββ Heading-only formats: each entry is (label, pattern, word_mode, roman_mode, titled) | |
| # titled=True means candidates include real title words β prefer over bare-number matches | |
| _FORMAT_TABLE = [ | |
| # ββ Highest priority: "Chapter N" variants WITH a title word after the number ββ | |
| # e.g. "Chapter 3: The Storm", "Ch.4 - Awakening", "Chapter 5 / Dawn", | |
| # "Chapter 5 | The Fall", "Chapter 5 Title" | |
| ("P1t", | |
| r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s*[-]?\s*\d+\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)", | |
| False, False, True), | |
| # "Chapter N" plain (no visible title after) β also matches "Chapter N: " with nothing | |
| ("P1", | |
| r"(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*)", | |
| False, False, False), | |
| # ββ "Chapter One / Ch. Twenty" etc. WITH title ββ | |
| ("P2t", | |
| r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s*[-]?\s*(?:" + _WORD_NUMS + r")\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)", | |
| True, False, True), | |
| ("P2", | |
| r"(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*(?:" + _WORD_NUMS + r")\b[^\n]*)", | |
| True, False, False), | |
| # ββ "Title (Chapter N)" β title comes BEFORE the chapter tag ββ | |
| ("P_tc", | |
| r"(?i)(?m)(^[ \t]*\S[^\n]+\((?:chapter|ch\.?)\s*\d+\)[ \t]*$)", | |
| False, False, True), | |
| # ββ Part / Book / Episode / Arc / Volume / Act + digit WITH title ββ | |
| ("P3t", | |
| r"(?i)(?m)(^[ \t]*(?:part|book|episode|arc|volume|vol\.?|act)\s*[-]?\s*\d+\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)", | |
| False, False, True), | |
| ("P3", | |
| r"(?i)(\b(?:part|book|episode|arc|volume|vol\.?|act)\s*[-:]?\s*\d+\b[^\n]*)", | |
| False, False, False), | |
| # ββ "Chapter [Word]" β e.g. "Chapter Prologue", "Chapter Epilogue" ββ | |
| ("P_cw", | |
| r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s+[A-Za-z][^\n]{1,60}$)", | |
| False, False, True), | |
| # ββ "N: Title" at line start (not timestamps: 1:30) ββ | |
| ("P4", | |
| r"(?m)(^[ \t]*\d+\s*:(?!\d)\s*\S[^\n]*)", | |
| False, False, True), | |
| # ββ "N - Title" at line start ββ | |
| ("P4d", | |
| r"(?m)(^[ \t]*\d+\s*-\s*\S[^\n]*)", | |
| False, False, True), | |
| # ββ "N | Title" at line start ββ | |
| ("P4p", | |
| r"(?m)(^[ \t]*\d+\s*\|\s*\S[^\n]*)", | |
| False, False, True), | |
| # ββ "N. Title" at line start ββ | |
| ("P5", | |
| r"(?m)(^[ \t]*\d+\.[ \t]+\S[^\n]*)", | |
| False, False, True), | |
| # ββ "N Title" at line start β bare space, no separator ββ | |
| # e.g. "3 The Storm", "12 New Dawn" | |
| # Guard: title word must start with a capital letter to avoid matching | |
| # pure numeric lines or timestamps; line capped at 80 chars (not a sentence) | |
| ("P4s", | |
| r"(?m)(^[ \t]*\d+[ \t]+[A-Z][^\n]{1,80}$)", | |
| False, False, True), | |
| # ββ Standalone Roman numeral on its own line ββ | |
| # Must be a valid Roman numeral (β₯ II to avoid ambiguity with "I") | |
| ("P_rom", | |
| r"(?m)(^[ \t]*(?:M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{1,3}))[ \t]*$)", | |
| False, True, False), | |
| # ββ Standalone Arabic numeral on its own line ββ | |
| ("P6", | |
| r"(?m)(^[ \t]*\d+[ \t]*$)", | |
| False, False, False), | |
| # ββ Standalone spelled-out number on its own line ββ | |
| ("P_word", | |
| r"(?im)(^[ \t]*(?:" + _WORD_NUMS + r")[ \t]*$)", | |
| True, False, False), | |
| # ββ "Section N" (last resort) ββ | |
| ("P7", | |
| r"(?i)(\bsection\s*[-:]?\s*\d+\b[^\n]*)", | |
| False, False, False), | |
| ] | |
| def _detect_and_split(full_text): | |
| """ | |
| Tries every format in _FORMAT_TABLE (priority order). | |
| Prefers formats whose candidates include real title words (titled=True). | |
| Within the same titled-ness, picks the first format that passes β₯60% ascending. | |
| Anti-false-positive guard: | |
| β’ Requires at least 3 candidates (2 is too easy to fake from random text). | |
| β’ Ignores a format if it only fires once deep inside the text (likely a bare reference). | |
| β’ For bare-number formats (P6, P_rom, P_word), requires the numbers to start near | |
| the top third of the document β real chapter lists do; random mid-text mentions don't. | |
| """ | |
| lines = full_text.count('\n') or 1 | |
| text_len = len(full_text) | |
| best = None # (titled, ratio, label, pattern, splits) | |
| for label, pattern, word_mode, roman_mode, titled in _FORMAT_TABLE: | |
| # Find all matches with their positions | |
| matches = list(re.finditer(pattern, full_text)) | |
| candidates = [m.group(1) for m in matches] | |
| min_count = 3 if label in ("P6", "P_rom", "P_word") else 2 | |
| if len(candidates) < min_count: | |
| continue | |
| # For bare-number formats, first match must be in the top 40% of the text | |
| if label in ("P6", "P_rom", "P_word"): | |
| first_pos = matches[0].start() | |
| if first_pos > text_len * 0.40: | |
| continue | |
| valid, ratio = _is_ascending(candidates, word_mode=word_mode, roman_mode=roman_mode) | |
| if not valid: | |
| continue | |
| # Keep this as a candidate; prefer titled=True over bare-number | |
| if best is None: | |
| best = (titled, ratio, label, pattern) | |
| else: | |
| prev_titled, prev_ratio, *_ = best | |
| # Titled always beats untitled | |
| if titled and not prev_titled: | |
| best = (titled, ratio, label, pattern) | |
| # Within same titled-ness, higher ratio wins | |
| elif titled == prev_titled and ratio > prev_ratio: | |
| best = (titled, ratio, label, pattern) | |
| # Stop early once we have a titled match with near-perfect ascending ratio | |
| if titled and ratio >= 0.95: | |
| break | |
| if best: | |
| _, _, _, pattern = best | |
| return re.split(pattern, full_text) | |
| # Absolute fallback | |
| return [full_text] | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # EPUB-SPECIFIC: extract chapters directly from HTML heading tags | |
| # βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _epub_extract_chapters_from_spine(zip_file, opf_soup, opf_dir, manifest): | |
| """ | |
| Walk the OPF spine in order. For each spine document, look for an | |
| <h1>/<h2>/<h3> heading that represents a chapter title. | |
| Priority rule: | |
| β’ A heading that contains "chapter" + a number/word is PREFERRED. | |
| β’ A heading with any word content beyond a bare number is still good. | |
| β’ A bare numeric-only heading is accepted as a fallback. | |
| β’ Headings that look like in-text sub-section markers (e.g. scene breaks) | |
| inside a file that already has a titled chapter heading are ignored. | |
| Returns a list of (title, body_text) pairs, spine-ordered. | |
| Duplicate consecutive chapter numbers (TOC artefact) are collapsed: | |
| the titled version is kept over the bare-number version. | |
| """ | |
| chapters = [] | |
| for itemref in opf_soup.find_all('itemref'): | |
| item_id = itemref.get('idref') | |
| if item_id not in manifest: | |
| continue | |
| href = manifest[item_id] | |
| file_path = href if not opf_dir else f"{opf_dir}/{href}" | |
| file_path = file_path.split('#')[0] | |
| try: | |
| html_data = zip_file.read(file_path) | |
| except KeyError: | |
| continue | |
| html_soup = BeautifulSoup(html_data, 'html.parser') | |
| # Collect ALL headings in this file | |
| headings = html_soup.find_all(['h1', 'h2', 'h3']) | |
| heading_texts = [h.get_text(strip=True) for h in headings if h.get_text(strip=True)] | |
| # Pick the BEST heading for this spine document | |
| chapter_title = None | |
| title_score = -1 | |
| for h_text in heading_texts: | |
| h_clean = h_text.strip() | |
| if not h_clean: | |
| continue | |
| score = 0 | |
| # Has "chapter" keyword? | |
| if re.search(r'(?i)\b(?:chapter|ch\.?)\b', h_clean): | |
| score += 3 | |
| # Has a real title word (beyond numbers/keywords)? | |
| if _title_has_words(h_clean): | |
| score += 2 | |
| # Has an Arabic number? | |
| if re.search(r'\d', h_clean): | |
| score += 1 | |
| # Has a Roman numeral? | |
| if re.search(r'\b' + _ROMAN_PAT + r'\b', h_clean, re.I): | |
| score += 1 | |
| if score > title_score: | |
| title_score = score | |
| chapter_title = h_clean | |
| # Extract full body text of this spine document | |
| body_text = html_soup.get_text(separator='\n').strip() | |
| if not body_text: | |
| continue | |
| if chapter_title: | |
| chapters.append((chapter_title, body_text)) | |
| else: | |
| # No heading found β attach to previous chapter as continuation | |
| if chapters: | |
| chapters[-1] = (chapters[-1][0], chapters[-1][1] + '\n' + body_text) | |
| # else: pre-chapter front matter; will become Intro | |
| if not chapters: | |
| return None # Signal: fall back to text-split method | |
| # ββ Collapse duplicate chapter numbers (TOC entry + real chapter) ββ | |
| # Keep the version with the higher title_score (titled over bare). | |
| deduped = [] | |
| for title, body in chapters: | |
| num = _extract_num(title, word_mode=True, roman_mode=True) | |
| if deduped and num is not None: | |
| prev_title, prev_body = deduped[-1] | |
| prev_num = _extract_num(prev_title, word_mode=True, roman_mode=True) | |
| if prev_num is not None and prev_num == num: | |
| # Keep the titled one; merge bodies | |
| pick_title = title if _title_has_words(title) else prev_title | |
| deduped[-1] = (pick_title, prev_body + '\n' + body) | |
| continue | |
| deduped.append((title, body)) | |
| # ββ Verify ascending order (β₯60%) ββ | |
| # If not, the EPUB had a non-sequential spine (alphabetically sorted files, etc.) | |
| # In that case we trust the order we have but warn | |
| nums = [_extract_num(t, word_mode=True, roman_mode=True) for t, _ in deduped] | |
| nums = [n for n in nums if n is not None] | |
| if len(nums) >= 2: | |
| ascending = sum(1 for i in range(1, len(nums)) if nums[i] > nums[i-1]) | |
| ratio = ascending / (len(nums) - 1) | |
| if ratio < 0.4: | |
| # Spine order looks scrambled; sort by extracted number | |
| numbered = [(t, b) for t, b in deduped if _extract_num(t, word_mode=True, roman_mode=True) is not None] | |
| unnumbered = [(t, b) for t, b in deduped if _extract_num(t, word_mode=True, roman_mode=True) is None] | |
| numbered.sort(key=lambda x: _extract_num(x[0], word_mode=True, roman_mode=True)) | |
| deduped = numbered + unnumbered | |
| return deduped if deduped else None | |
| def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books): | |
| text_file, epub_chapters = extract_local_text(file_path, temp_dir) | |
| if not text_file and epub_chapters is None: return None, None, None | |
| base_name = os.path.splitext(os.path.basename(file_path))[0] | |
| book_label = f"[Book {book_idx}/{total_books}]" | |
| # ββ 1. PARSE CHAPTERS βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| chapters = [] | |
| if epub_chapters: | |
| # ββ EPUB heading-based path ββββββββββββββββββββββββββββββββββββββββββ | |
| # epub_chapters is already a list of (title, body_text) in spine order | |
| print(f"π {book_label} Using EPUB heading extraction ({len(epub_chapters)} chapters found)") | |
| prev_num = 0 | |
| for title, body in epub_chapters: | |
| body_clean = body.replace('\n', ' ').strip() | |
| title_clean = title.strip() | |
| current_num = _extract_num(title_clean, word_mode=True, roman_mode=True) | |
| # Out-of-order: backward reference / in-text chapter mention | |
| if current_num is not None and prev_num > 0 and current_num < prev_num: | |
| if chapters: | |
| chapters[-1][1] += f" {title_clean}. {body_clean}" | |
| continue | |
| # Double heading: same number as last accepted chapter | |
| if current_num is not None and chapters: | |
| last_num = _extract_num(chapters[-1][0], word_mode=True, roman_mode=True) | |
| if last_num is not None and last_num == current_num: | |
| # Keep the titled version as the chapter name | |
| if _title_has_words(title_clean) and not _title_has_words(chapters[-1][0]): | |
| chapters[-1][0] = title_clean | |
| chapters[-1][1] += f" {body_clean}" | |
| continue | |
| # Strip the heading from the start of body_clean to avoid repeating | |
| # the chapter title (get_text() already includes the <h1/h2/h3> text). | |
| body_no_title = body_clean | |
| if body_clean.lower().startswith(title_clean.lower()): | |
| body_no_title = body_clean[len(title_clean):].lstrip(". \t") | |
| # TOC / tiny body β merge into previous | |
| if chapters and len(chapters[-1][1]) < 150: | |
| chapters[-1][1] += f" {title_clean}. {body_no_title}" | |
| chapters[-1][0] = title_clean | |
| continue | |
| chapters.append([title_clean, f"{title_clean}. {body_no_title}"]) | |
| if current_num is not None: | |
| prev_num = current_num | |
| else: | |
| # ββ Text/PDF/EPUB-fallback path ββββββββββββββββββββββββββββββββββββββ | |
| if not text_file: | |
| return None, None, None | |
| with open(text_file, 'r', encoding='utf-8') as f: | |
| full_text = f.read() | |
| splits = _detect_and_split(full_text) | |
| prev_num = 0 | |
| if len(splits) > 1 and splits[0].strip(): | |
| chapters.append(["Intro", splits[0].replace('\n', ' ').strip()]) | |
| for i in range(1, len(splits), 2): | |
| title = splits[i].strip() | |
| body = splits[i + 1].strip() if i + 1 < len(splits) else "" | |
| # SAFEGUARD: title grabbed a whole paragraph | |
| if len(title) > 100: | |
| body = title[100:] + " " + body | |
| title = title[:100] + "..." | |
| current_num = _extract_num(title, word_mode=True, roman_mode=True) | |
| # Out-of-order: backward reference | |
| if current_num is not None and prev_num > 0 and current_num < prev_num: | |
| if chapters: | |
| chapters[-1][1] += f" {title}. {body}".replace('\n', ' ') | |
| continue | |
| # Double heading: same number as last accepted chapter | |
| if current_num is not None and chapters: | |
| last_num = _extract_num(chapters[-1][0], word_mode=True, roman_mode=True) | |
| if last_num is not None and last_num == current_num: | |
| chapters[-1][1] += f" {body}".replace('\n', ' ') | |
| continue | |
| # TOC / tiny body | |
| if chapters and len(chapters[-1][1]) < 150: | |
| chapters[-1][1] += f" {title}. {body}".replace('\n', ' ') | |
| chapters[-1][0] = title | |
| continue | |
| chapters.append([title, f"{title}. {body}".replace('\n', ' ')]) | |
| if current_num is not None: | |
| prev_num = current_num | |
| if not chapters or (len(chapters) == 1 and chapters[0][0] == 'Intro'): | |
| # Ultimate fallback: treat entire text as one chapter | |
| if text_file: | |
| with open(text_file, 'r', encoding='utf-8') as f: | |
| fallback_text = f.read() | |
| elif epub_chapters: | |
| fallback_text = ' '.join(b for _, b in epub_chapters) | |
| else: | |
| fallback_text = "" | |
| chapters = [["Full Book", fallback_text.replace('\n', ' ')]] | |
| engine = GoogleTTSEngine() | |
| all_chunks = [] | |
| for c_idx, (title, c_text) in enumerate(chapters): | |
| c_chunks = [c for c in engine.split_text(c_text) if c.strip()] | |
| for chunk in c_chunks: | |
| all_chunks.append((c_idx, chunk)) | |
| total_chunks = len(all_chunks) | |
| chunk_dir = os.path.join(temp_dir, "chunks") | |
| os.makedirs(chunk_dir, exist_ok=True) | |
| # 2. DOWNLOAD CHUNKS | |
| start_time = time.time() | |
| completed = 0 | |
| with ThreadPoolExecutor(max_workers=THREADS) as pool: | |
| futures = {} | |
| c_chunk_counters = {c_idx: 0 for c_idx in range(len(chapters))} | |
| for c_idx, chunk in all_chunks: | |
| f = pool.submit(engine.download_chunk, chunk) | |
| futures[f] = (c_idx, c_chunk_counters[c_idx]) | |
| c_chunk_counters[c_idx] += 1 | |
| for f in as_completed(futures): | |
| c_idx, local_c_idx = futures[f] | |
| data = f.result() | |
| if data: | |
| out_path = os.path.join(chunk_dir, f"c{c_idx:03d}_{local_c_idx:05d}.mp3") | |
| with open(out_path, "wb") as af: af.write(data) | |
| completed += 1 | |
| elapsed = time.time() - start_time | |
| avg = elapsed / completed | |
| eta = avg * (total_chunks - completed) | |
| progress_obj((completed/total_chunks), desc=f"π₯ {book_label} Downloading: {completed}/{total_chunks} | β³ ETA: {int(eta//60):02d}:{int(eta%60):02d}") | |
| # 3. MERGE AND MEASURE PER CHAPTER | |
| processed_chapters = [] | |
| metadata_entries = [] | |
| current_time_ms = 0 | |
| for c_idx, (title, _) in enumerate(chapters): | |
| c_files = sorted([f for f in os.listdir(chunk_dir) if f.startswith(f"c{c_idx:03d}_")]) | |
| if not c_files: continue | |
| raw_c_path = os.path.join(temp_dir, f"raw_c{c_idx}.mp3") | |
| with open(raw_c_path, 'wb') as outfile: | |
| for cf in c_files: | |
| with open(os.path.join(chunk_dir, cf), 'rb') as infile: | |
| shutil.copyfileobj(infile, outfile) | |
| proc_c_path = os.path.join(temp_dir, f"proc_c{c_idx}.mp3") | |
| subprocess.run([ | |
| "ffmpeg", "-y", "-i", raw_c_path, | |
| "-af", f"silenceremove=1:0:-50dB,atempo={FIXED_SPEED}", | |
| "-threads", "0", proc_c_path | |
| ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) | |
| processed_chapters.append(proc_c_path) | |
| # Determine the duration of this specific chapter to build precise metadata | |
| dur_str = subprocess.run([ | |
| "ffprobe", "-v", "error", "-show_entries", "format=duration", | |
| "-of", "default=noprint_wrappers=1:nokey=1", proc_c_path | |
| ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout | |
| try: | |
| duration_ms = int(float(dur_str.decode().strip()) * 1000) | |
| except (ValueError, AttributeError): | |
| duration_ms = 0 | |
| metadata_entries.append({ | |
| "title": title, | |
| "start": current_time_ms, | |
| "end": current_time_ms + duration_ms | |
| }) | |
| current_time_ms += duration_ms | |
| return processed_chapters, metadata_entries, base_name | |
| def final_merge_and_metadata(processed_chapters, metadata_entries, base_name, temp_dir): | |
| # 1. Create FFMETADATA file | |
| meta_path = os.path.join(temp_dir, "metadata.txt") | |
| with open(meta_path, "w", encoding="utf-8") as f: | |
| f.write(";FFMETADATA1\n") | |
| f.write(f"title={base_name}\n\n") | |
| for entry in metadata_entries: | |
| f.write("[CHAPTER]\n") | |
| f.write("TIMEBASE=1/1000\n") | |
| f.write(f"START={entry['start']}\n") | |
| f.write(f"END={entry['end']}\n") | |
| f.write(f"title={entry['title']}\n\n") | |
| # 2. Create concat demuxer file | |
| concat_path = os.path.join(temp_dir, "concat.txt") | |
| with open(concat_path, "w", encoding="utf-8") as f: | |
| for p in processed_chapters: | |
| # FFmpeg requires forward slashes, even on Windows | |
| f.write(f"file '{os.path.abspath(p).replace(chr(92), '/')}'\n") | |
| # 3. Generate outputs | |
| output_mp3 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp3") | |
| output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4") | |
| # Merge to MP3 (Fast Stream Copy) | |
| r_mp3 = subprocess.run([ | |
| "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path, | |
| "-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3 | |
| ], capture_output=True, text=True) | |
| if r_mp3.returncode != 0: | |
| print(f"\u26a0\ufe0f ffmpeg MP3 merge failed for {base_name}:\n{r_mp3.stderr[-800:]}") | |
| # Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility) | |
| r_mp4 = subprocess.run([ | |
| "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path, | |
| "-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4 | |
| ], capture_output=True, text=True) | |
| if r_mp4.returncode != 0: | |
| print(f"\u26a0\ufe0f ffmpeg MP4 merge failed for {base_name}:\n{r_mp4.stderr[-800:]}") | |
| shutil.rmtree(temp_dir) | |
| # Only return files that were actually created | |
| ready = [p for p in [output_mp3, output_mp4] if os.path.isfile(p) and os.path.getsize(p) > 0] | |
| if not ready: | |
| print(f"\u26a0\ufe0f {base_name}: both output files missing after ffmpeg \u2014 bucket sync skipped") | |
| return ready | |
| async def process_single_book(file_path, progress_obj, book_idx, total_books): | |
| temp_dir = tempfile.mkdtemp() | |
| async with download_door: | |
| processed_chapters, metadata_entries, base_name = await asyncio.to_thread( | |
| sync_process_book, file_path, temp_dir, progress_obj, book_idx, total_books | |
| ) | |
| if not processed_chapters: return [] | |
| async with kitchen_door: | |
| output_files = await asyncio.to_thread( | |
| final_merge_and_metadata, processed_chapters, metadata_entries, base_name, temp_dir | |
| ) | |
| # ββ Push to HF bucket ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| book_label = f"[Book {book_idx}/{total_books}]" | |
| await asyncio.to_thread(sync_to_hf_bucket, output_files, book_label) | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| return output_files | |
| async def process_books_holygrail(file_paths, progress=gr.Progress()): | |
| if not file_paths: | |
| yield [] | |
| return | |
| if not isinstance(file_paths, list): file_paths = [file_paths] | |
| total = len(file_paths) | |
| tasks = [asyncio.create_task(process_single_book(f, progress, idx+1, total)) for idx, f in enumerate(file_paths)] | |
| finished_files = [] | |
| for coro in asyncio.as_completed(tasks): | |
| results = await coro | |
| if results: | |
| finished_files.extend(results) | |
| yield finished_files | |
| if len(finished_files) > 2: # More than 2 files means multiple books processed | |
| zip_path = os.path.join(tempfile.gettempdir(), "AudioNovel_MasterBatch.zip") | |
| with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: | |
| for f in finished_files: zipf.write(f, os.path.basename(f)) | |
| # Sync the master ZIP to HF bucket as well | |
| await asyncio.to_thread(sync_to_hf_bucket, [zip_path], "[Batch ZIP]") | |
| finished_files.insert(0, zip_path) | |
| yield finished_files | |
| with gr.Blocks(theme=gr.themes.Soft(primary_hue="indigo")) as app: | |
| gr.Markdown("# π§ AudioNovel Studio (Smooth-Flow Edition)") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| file_in = gr.File(label="π Upload Books", file_count="multiple") | |
| with gr.Row(): | |
| start_btn = gr.Button("π Start Pipeline", variant="primary") | |
| stop_btn = gr.Button("π Stop", variant="stop") | |
| with gr.Column(scale=1): | |
| file_out = gr.File(label="π₯ Live Download House (MP3 & MP4)", file_count="multiple", interactive=False) | |
| process_event = start_btn.click(fn=process_books_holygrail, inputs=file_in, outputs=file_out) | |
| stop_btn.click(fn=None, inputs=None, outputs=None, cancels=[process_event]) | |
| if __name__ == "__main__": | |
| app.launch() |