import gradio as gr
import os
import tempfile
import requests
import subprocess
import zipfile
import shutil
import time
import asyncio
import re
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
from huggingface_hub import HfFileSystem
# --- CONFIGURATION ---
FIXED_SPEED = 1.6
THREADS = 10
# --- HF BUCKET ---
HF_BUCKET = "hf://buckets/Krixter/storage"
HF_LOCAL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool:
"""Upload finished audio files directly to the HF bucket via HfFileSystem."""
token = os.environ.get("HF_TOKEN")
if not token:
print("⚠️ HF_TOKEN secret not set — bucket upload skipped.")
return False
try:
fs = HfFileSystem(token=token)
uploaded = 0
for src in files:
if not src or not os.path.isfile(src):
continue
dest = f"{HF_BUCKET}/{os.path.basename(src)}"
print(f"📤 {book_label} Uploading {os.path.basename(src)} → {dest}")
fs.put_file(src, dest)
print(f"☁️ {book_label} Uploaded OK: {os.path.basename(src)}")
uploaded += 1
return uploaded > 0
except Exception as e:
print(f"⚠️ {book_label} Bucket upload failed: {e}")
return False
# --- THE BOUNCERS ---
download_door = asyncio.Semaphore(1)
kitchen_door = asyncio.Semaphore(1)
def extract_local_text(path, temp_dir):
"""
Returns (text_file_path, epub_chapters_or_None).
epub_chapters is a list of (title, body_text) pairs when the file is an EPUB
and heading-based chapter extraction succeeded. In all other cases it is None,
and the caller should use detect_and_split() on the text file.
"""
content = ""
epub_chapters = None # ← NEW: populated only for EPUBs
ext = os.path.splitext(path)[1].lower()
try:
if ext == '.txt':
with open(path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
# ── BULLETPROOF EPUB BLOCK ────────────────────────────────────────────
elif ext == '.epub':
with zipfile.ZipFile(path, 'r') as z:
try:
# 1. Find the master OPF file
container = z.read('META-INF/container.xml')
c_soup = BeautifulSoup(container, 'html.parser')
opf_path = c_soup.find('rootfile').get('full-path')
opf_dir = os.path.dirname(opf_path)
opf_content = z.read(opf_path)
opf_soup = BeautifulSoup(opf_content, 'html.parser')
# 2. Build the Manifest
manifest = {}
for item in opf_soup.find_all('item'):
manifest[item.get('id')] = item.get('href')
# 3. Try heading-based chapter extraction (NEW)
epub_chapters = _epub_extract_chapters_from_spine(z, opf_soup, opf_dir, manifest)
# 4. Always build the flat text too (needed as fallback)
for itemref in opf_soup.find_all('itemref'):
item_id = itemref.get('idref')
if item_id not in manifest:
continue
href = manifest[item_id]
file_path = href if not opf_dir else f"{opf_dir}/{href}"
file_path = file_path.split('#')[0]
try:
html_data = z.read(file_path)
html_soup = BeautifulSoup(html_data, 'html.parser')
clean_part = html_soup.get_text(separator='\n')
if clean_part:
content += clean_part + "\n"
except KeyError:
continue
except Exception as e:
print(f"⚠️ Spine/heading parse failed, using fallback mode: {e}")
epub_chapters = None
for filename in z.namelist():
if filename.endswith(('.html', '.xhtml')):
with z.open(filename) as f:
sb = BeautifulSoup(f.read(), 'html.parser')
cp = sb.get_text(separator='\n')
if cp:
content += cp + "\n"
# ── END EPUB BLOCK ────────────────────────────────────────────────────
elif ext == '.pdf':
import PyPDF2
reader = PyPDF2.PdfReader(path)
for page in reader.pages:
txt = page.extract_text()
if txt:
content += txt + "\n"
except Exception as e:
return None, None
if not content and epub_chapters is None:
return None, None
# Clean up horizontal whitespace; preserve newlines
content = re.sub(r'[ \t]+', ' ', content)
content = re.sub(r'\n\s*\n', '\n\n', content)
out = os.path.join(temp_dir, "extracted.txt")
with open(out, 'w', encoding='utf-8') as f:
f.write(content)
return out, epub_chapters
class GoogleTTSEngine:
def split_text(self, text, max_len=200):
"""SMART SPLIT: Prioritizes cutting at sentence endings to reduce robotic pauses."""
chunks = []
while text:
if len(text) <= max_len:
chunks.append(text.strip())
break
best_i = -1
for punct in [". ", "! ", "? ", "; "]:
i = text[:max_len].rfind(punct)
if i > best_i:
best_i = i
if best_i == -1:
best_i = text[:max_len].rfind(" ")
if best_i == -1:
best_i = max_len
else:
best_i += 1
chunks.append(text[:best_i].strip())
text = text[best_i:].lstrip()
return chunks
def strip_id3(self, mp3_data):
if len(mp3_data) >= 10 and mp3_data[:3] == b"ID3":
size = ((mp3_data[6] & 0x7F) << 21 | (mp3_data[7] & 0x7F) << 14 |
(mp3_data[8] & 0x7F) << 7 | (mp3_data[9] & 0x7F))
mp3_data = mp3_data[10 + size:]
if len(mp3_data) > 128 and mp3_data[-128:-125] == b"TAG":
mp3_data = mp3_data[:-128]
return mp3_data
def download_chunk(self, text):
url = "https://translate.google.com/translate_tts"
params = {"ie": "UTF-8", "q": text, "tl": "en-IN", "client": "tw-ob"}
headers = {"User-Agent": "Mozilla/5.0"}
for _ in range(5):
try:
r = requests.get(url, params=params, headers=headers, timeout=5)
if r.status_code == 200: return self.strip_id3(r.content)
except: time.sleep(0.5)
return None
# ─────────────────────────────────────────────────────────────────────────────
# CHAPTER DETECTION ENGINE
# ─────────────────────────────────────────────────────────────────────────────
_WORD_NUMS = (
"zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|"
"thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|"
"thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred"
)
# Roman numeral pattern (standalone, not substrings like "VI" inside a word)
_ROMAN_PAT = r"(?:M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3}))"
def _roman_to_int(s):
"""Convert a Roman numeral string to an integer (returns 0 on failure)."""
vals = {'I':1,'V':5,'X':10,'L':50,'C':100,'D':500,'M':1000}
s = s.upper().strip()
if not s:
return 0
result, prev = 0, 0
for ch in reversed(s):
v = vals.get(ch, 0)
if v < prev:
result -= v
else:
result += v
prev = v
return result
_WORD_NUM_MAP = {w: i for i, w in enumerate(_WORD_NUMS.split("|"))}
def _extract_num(text, word_mode=False, roman_mode=False):
"""
Pull a chapter sequence number out of a heading string.
Tries Arabic → Roman → Word in that order (unless a specific mode is set).
Returns an integer or None.
"""
# Arabic first — always tried
m = re.search(r"\d+", text)
if m:
return int(m.group())
if roman_mode:
m = re.search(r"\b(" + _ROMAN_PAT + r")\b", text, re.I)
if m and m.group(1).upper() != "I": # lone "I" is too ambiguous
return _roman_to_int(m.group(1))
if word_mode:
m = re.search(r"\b(" + _WORD_NUMS + r")\b", text, re.I)
if m:
return _WORD_NUM_MAP.get(m.group(1).lower(), -1)
return None
def _is_ascending(candidates, word_mode=False, roman_mode=False):
"""
Return (is_valid, ratio).
Valid when ≥60 % of consecutive pairs are strictly ascending.
Deduplicates runs of the same number first (EPUB double-heading artefact).
Skips un-parseable candidates rather than failing.
Requires at least 2 parseable, distinct numbers.
"""
nums = []
for c in candidates:
n = _extract_num(c, word_mode=word_mode, roman_mode=roman_mode)
if n is not None:
nums.append(n)
# Deduplicate consecutive equal numbers (e.g. 1,1,2,2,3,3 → 1,2,3)
deduped = []
for n in nums:
if not deduped or n != deduped[-1]:
deduped.append(n)
if len(deduped) < 2:
return False, 0.0
ascending = sum(1 for j in range(1, len(deduped)) if deduped[j] > deduped[j-1])
ratio = ascending / (len(deduped) - 1)
return ratio >= 0.6, ratio
def _title_has_words(heading):
"""True if the heading contains non-numeric, non-separator words beyond the keyword."""
# Strip common chapter-keywords and numbers, see if real words remain
cleaned = re.sub(
r"(?i)\b(?:chapter|ch\.?|part|book|episode|arc|volume|vol\.?|act|section)\b", "", heading
)
cleaned = re.sub(r"[\d\s\-:./|()]+", " ", cleaned).strip()
return bool(cleaned)
# ── Heading-only formats: each entry is (label, pattern, word_mode, roman_mode, titled)
# titled=True means candidates include real title words → prefer over bare-number matches
_FORMAT_TABLE = [
# ── Highest priority: "Chapter N" variants WITH a title word after the number ──
# e.g. "Chapter 3: The Storm", "Ch.4 - Awakening", "Chapter 5 / Dawn",
# "Chapter 5 | The Fall", "Chapter 5 Title"
("P1t",
r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s*[-]?\s*\d+\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)",
False, False, True),
# "Chapter N" plain (no visible title after) — also matches "Chapter N: " with nothing
("P1",
r"(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*)",
False, False, False),
# ── "Chapter One / Ch. Twenty" etc. WITH title ──
("P2t",
r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s*[-]?\s*(?:" + _WORD_NUMS + r")\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)",
True, False, True),
("P2",
r"(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*(?:" + _WORD_NUMS + r")\b[^\n]*)",
True, False, False),
# ── "Title (Chapter N)" — title comes BEFORE the chapter tag ──
("P_tc",
r"(?i)(?m)(^[ \t]*\S[^\n]+\((?:chapter|ch\.?)\s*\d+\)[ \t]*$)",
False, False, True),
# ── Part / Book / Episode / Arc / Volume / Act + digit WITH title ──
("P3t",
r"(?i)(?m)(^[ \t]*(?:part|book|episode|arc|volume|vol\.?|act)\s*[-]?\s*\d+\s*(?:[:.\-|/]\s*|\s+)\S[^\n]+)",
False, False, True),
("P3",
r"(?i)(\b(?:part|book|episode|arc|volume|vol\.?|act)\s*[-:]?\s*\d+\b[^\n]*)",
False, False, False),
# ── "Chapter [Word]" — e.g. "Chapter Prologue", "Chapter Epilogue" ──
("P_cw",
r"(?i)(?m)(^[ \t]*(?:chapter|ch\.?)\s+[A-Za-z][^\n]{1,60}$)",
False, False, True),
# ── "N: Title" at line start (not timestamps: 1:30) ──
("P4",
r"(?m)(^[ \t]*\d+\s*:(?!\d)\s*\S[^\n]*)",
False, False, True),
# ── "N - Title" at line start ──
("P4d",
r"(?m)(^[ \t]*\d+\s*-\s*\S[^\n]*)",
False, False, True),
# ── "N | Title" at line start ──
("P4p",
r"(?m)(^[ \t]*\d+\s*\|\s*\S[^\n]*)",
False, False, True),
# ── "N. Title" at line start ──
("P5",
r"(?m)(^[ \t]*\d+\.[ \t]+\S[^\n]*)",
False, False, True),
# ── "N Title" at line start — bare space, no separator ──
# e.g. "3 The Storm", "12 New Dawn"
# Guard: title word must start with a capital letter to avoid matching
# pure numeric lines or timestamps; line capped at 80 chars (not a sentence)
("P4s",
r"(?m)(^[ \t]*\d+[ \t]+[A-Z][^\n]{1,80}$)",
False, False, True),
# ── Standalone Roman numeral on its own line ──
# Must be a valid Roman numeral (≥ II to avoid ambiguity with "I")
("P_rom",
r"(?m)(^[ \t]*(?:M{0,4}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{1,3}))[ \t]*$)",
False, True, False),
# ── Standalone Arabic numeral on its own line ──
("P6",
r"(?m)(^[ \t]*\d+[ \t]*$)",
False, False, False),
# ── Standalone spelled-out number on its own line ──
("P_word",
r"(?im)(^[ \t]*(?:" + _WORD_NUMS + r")[ \t]*$)",
True, False, False),
# ── "Section N" (last resort) ──
("P7",
r"(?i)(\bsection\s*[-:]?\s*\d+\b[^\n]*)",
False, False, False),
]
def _detect_and_split(full_text):
"""
Tries every format in _FORMAT_TABLE (priority order).
Prefers formats whose candidates include real title words (titled=True).
Within the same titled-ness, picks the first format that passes ≥60% ascending.
Anti-false-positive guard:
• Requires at least 3 candidates (2 is too easy to fake from random text).
• Ignores a format if it only fires once deep inside the text (likely a bare reference).
• For bare-number formats (P6, P_rom, P_word), requires the numbers to start near
the top third of the document — real chapter lists do; random mid-text mentions don't.
"""
lines = full_text.count('\n') or 1
text_len = len(full_text)
best = None # (titled, ratio, label, pattern, splits)
for label, pattern, word_mode, roman_mode, titled in _FORMAT_TABLE:
# Find all matches with their positions
matches = list(re.finditer(pattern, full_text))
candidates = [m.group(1) for m in matches]
min_count = 3 if label in ("P6", "P_rom", "P_word") else 2
if len(candidates) < min_count:
continue
# For bare-number formats, first match must be in the top 40% of the text
if label in ("P6", "P_rom", "P_word"):
first_pos = matches[0].start()
if first_pos > text_len * 0.40:
continue
valid, ratio = _is_ascending(candidates, word_mode=word_mode, roman_mode=roman_mode)
if not valid:
continue
# Keep this as a candidate; prefer titled=True over bare-number
if best is None:
best = (titled, ratio, label, pattern)
else:
prev_titled, prev_ratio, *_ = best
# Titled always beats untitled
if titled and not prev_titled:
best = (titled, ratio, label, pattern)
# Within same titled-ness, higher ratio wins
elif titled == prev_titled and ratio > prev_ratio:
best = (titled, ratio, label, pattern)
# Stop early once we have a titled match with near-perfect ascending ratio
if titled and ratio >= 0.95:
break
if best:
_, _, _, pattern = best
return re.split(pattern, full_text)
# Absolute fallback
return [full_text]
# ─────────────────────────────────────────────────────────────────────────────
# EPUB-SPECIFIC: extract chapters directly from HTML heading tags
# ─────────────────────────────────────────────────────────────────────────────
def _epub_extract_chapters_from_spine(zip_file, opf_soup, opf_dir, manifest):
"""
Walk the OPF spine in order. For each spine document, look for an
// heading that represents a chapter title.
Priority rule:
• A heading that contains "chapter" + a number/word is PREFERRED.
• A heading with any word content beyond a bare number is still good.
• A bare numeric-only heading is accepted as a fallback.
• Headings that look like in-text sub-section markers (e.g. scene breaks)
inside a file that already has a titled chapter heading are ignored.
Returns a list of (title, body_text) pairs, spine-ordered.
Duplicate consecutive chapter numbers (TOC artefact) are collapsed:
the titled version is kept over the bare-number version.
"""
chapters = []
for itemref in opf_soup.find_all('itemref'):
item_id = itemref.get('idref')
if item_id not in manifest:
continue
href = manifest[item_id]
file_path = href if not opf_dir else f"{opf_dir}/{href}"
file_path = file_path.split('#')[0]
try:
html_data = zip_file.read(file_path)
except KeyError:
continue
html_soup = BeautifulSoup(html_data, 'html.parser')
# Collect ALL headings in this file
headings = html_soup.find_all(['h1', 'h2', 'h3'])
heading_texts = [h.get_text(strip=True) for h in headings if h.get_text(strip=True)]
# Pick the BEST heading for this spine document
chapter_title = None
title_score = -1
for h_text in heading_texts:
h_clean = h_text.strip()
if not h_clean:
continue
score = 0
# Has "chapter" keyword?
if re.search(r'(?i)\b(?:chapter|ch\.?)\b', h_clean):
score += 3
# Has a real title word (beyond numbers/keywords)?
if _title_has_words(h_clean):
score += 2
# Has an Arabic number?
if re.search(r'\d', h_clean):
score += 1
# Has a Roman numeral?
if re.search(r'\b' + _ROMAN_PAT + r'\b', h_clean, re.I):
score += 1
if score > title_score:
title_score = score
chapter_title = h_clean
# Extract full body text of this spine document
body_text = html_soup.get_text(separator='\n').strip()
if not body_text:
continue
if chapter_title:
chapters.append((chapter_title, body_text))
else:
# No heading found — attach to previous chapter as continuation
if chapters:
chapters[-1] = (chapters[-1][0], chapters[-1][1] + '\n' + body_text)
# else: pre-chapter front matter; will become Intro
if not chapters:
return None # Signal: fall back to text-split method
# ── Collapse duplicate chapter numbers (TOC entry + real chapter) ──
# Keep the version with the higher title_score (titled over bare).
deduped = []
for title, body in chapters:
num = _extract_num(title, word_mode=True, roman_mode=True)
if deduped and num is not None:
prev_title, prev_body = deduped[-1]
prev_num = _extract_num(prev_title, word_mode=True, roman_mode=True)
if prev_num is not None and prev_num == num:
# Keep the titled one; merge bodies
pick_title = title if _title_has_words(title) else prev_title
deduped[-1] = (pick_title, prev_body + '\n' + body)
continue
deduped.append((title, body))
# ── Verify ascending order (≥60%) ──
# If not, the EPUB had a non-sequential spine (alphabetically sorted files, etc.)
# In that case we trust the order we have but warn
nums = [_extract_num(t, word_mode=True, roman_mode=True) for t, _ in deduped]
nums = [n for n in nums if n is not None]
if len(nums) >= 2:
ascending = sum(1 for i in range(1, len(nums)) if nums[i] > nums[i-1])
ratio = ascending / (len(nums) - 1)
if ratio < 0.4:
# Spine order looks scrambled; sort by extracted number
numbered = [(t, b) for t, b in deduped if _extract_num(t, word_mode=True, roman_mode=True) is not None]
unnumbered = [(t, b) for t, b in deduped if _extract_num(t, word_mode=True, roman_mode=True) is None]
numbered.sort(key=lambda x: _extract_num(x[0], word_mode=True, roman_mode=True))
deduped = numbered + unnumbered
return deduped if deduped else None
def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
text_file, epub_chapters = extract_local_text(file_path, temp_dir)
if not text_file and epub_chapters is None: return None, None, None
base_name = os.path.splitext(os.path.basename(file_path))[0]
book_label = f"[Book {book_idx}/{total_books}]"
# ── 1. PARSE CHAPTERS ─────────────────────────────────────────────────────
chapters = []
if epub_chapters:
# ── EPUB heading-based path ──────────────────────────────────────────
# epub_chapters is already a list of (title, body_text) in spine order
print(f"📖 {book_label} Using EPUB heading extraction ({len(epub_chapters)} chapters found)")
prev_num = 0
for title, body in epub_chapters:
body_clean = body.replace('\n', ' ').strip()
title_clean = title.strip()
current_num = _extract_num(title_clean, word_mode=True, roman_mode=True)
# Out-of-order: backward reference / in-text chapter mention
if current_num is not None and prev_num > 0 and current_num < prev_num:
if chapters:
chapters[-1][1] += f" {title_clean}. {body_clean}"
continue
# Double heading: same number as last accepted chapter
if current_num is not None and chapters:
last_num = _extract_num(chapters[-1][0], word_mode=True, roman_mode=True)
if last_num is not None and last_num == current_num:
# Keep the titled version as the chapter name
if _title_has_words(title_clean) and not _title_has_words(chapters[-1][0]):
chapters[-1][0] = title_clean
chapters[-1][1] += f" {body_clean}"
continue
# Strip the heading from the start of body_clean to avoid repeating
# the chapter title (get_text() already includes the text).
body_no_title = body_clean
if body_clean.lower().startswith(title_clean.lower()):
body_no_title = body_clean[len(title_clean):].lstrip(". \t")
# TOC / tiny body — merge into previous
if chapters and len(chapters[-1][1]) < 150:
chapters[-1][1] += f" {title_clean}. {body_no_title}"
chapters[-1][0] = title_clean
continue
chapters.append([title_clean, f"{title_clean}. {body_no_title}"])
if current_num is not None:
prev_num = current_num
else:
# ── Text/PDF/EPUB-fallback path ──────────────────────────────────────
if not text_file:
return None, None, None
with open(text_file, 'r', encoding='utf-8') as f:
full_text = f.read()
splits = _detect_and_split(full_text)
prev_num = 0
if len(splits) > 1 and splits[0].strip():
chapters.append(["Intro", splits[0].replace('\n', ' ').strip()])
for i in range(1, len(splits), 2):
title = splits[i].strip()
body = splits[i + 1].strip() if i + 1 < len(splits) else ""
# SAFEGUARD: title grabbed a whole paragraph
if len(title) > 100:
body = title[100:] + " " + body
title = title[:100] + "..."
current_num = _extract_num(title, word_mode=True, roman_mode=True)
# Out-of-order: backward reference
if current_num is not None and prev_num > 0 and current_num < prev_num:
if chapters:
chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
continue
# Double heading: same number as last accepted chapter
if current_num is not None and chapters:
last_num = _extract_num(chapters[-1][0], word_mode=True, roman_mode=True)
if last_num is not None and last_num == current_num:
chapters[-1][1] += f" {body}".replace('\n', ' ')
continue
# TOC / tiny body
if chapters and len(chapters[-1][1]) < 150:
chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
chapters[-1][0] = title
continue
chapters.append([title, f"{title}. {body}".replace('\n', ' ')])
if current_num is not None:
prev_num = current_num
if not chapters or (len(chapters) == 1 and chapters[0][0] == 'Intro'):
# Ultimate fallback: treat entire text as one chapter
if text_file:
with open(text_file, 'r', encoding='utf-8') as f:
fallback_text = f.read()
elif epub_chapters:
fallback_text = ' '.join(b for _, b in epub_chapters)
else:
fallback_text = ""
chapters = [["Full Book", fallback_text.replace('\n', ' ')]]
engine = GoogleTTSEngine()
all_chunks = []
for c_idx, (title, c_text) in enumerate(chapters):
c_chunks = [c for c in engine.split_text(c_text) if c.strip()]
for chunk in c_chunks:
all_chunks.append((c_idx, chunk))
total_chunks = len(all_chunks)
chunk_dir = os.path.join(temp_dir, "chunks")
os.makedirs(chunk_dir, exist_ok=True)
# 2. DOWNLOAD CHUNKS
start_time = time.time()
completed = 0
with ThreadPoolExecutor(max_workers=THREADS) as pool:
futures = {}
c_chunk_counters = {c_idx: 0 for c_idx in range(len(chapters))}
for c_idx, chunk in all_chunks:
f = pool.submit(engine.download_chunk, chunk)
futures[f] = (c_idx, c_chunk_counters[c_idx])
c_chunk_counters[c_idx] += 1
for f in as_completed(futures):
c_idx, local_c_idx = futures[f]
data = f.result()
if data:
out_path = os.path.join(chunk_dir, f"c{c_idx:03d}_{local_c_idx:05d}.mp3")
with open(out_path, "wb") as af: af.write(data)
completed += 1
elapsed = time.time() - start_time
avg = elapsed / completed
eta = avg * (total_chunks - completed)
progress_obj((completed/total_chunks), desc=f"📥 {book_label} Downloading: {completed}/{total_chunks} | ⏳ ETA: {int(eta//60):02d}:{int(eta%60):02d}")
# 3. MERGE AND MEASURE PER CHAPTER
processed_chapters = []
metadata_entries = []
current_time_ms = 0
for c_idx, (title, _) in enumerate(chapters):
c_files = sorted([f for f in os.listdir(chunk_dir) if f.startswith(f"c{c_idx:03d}_")])
if not c_files: continue
raw_c_path = os.path.join(temp_dir, f"raw_c{c_idx}.mp3")
with open(raw_c_path, 'wb') as outfile:
for cf in c_files:
with open(os.path.join(chunk_dir, cf), 'rb') as infile:
shutil.copyfileobj(infile, outfile)
proc_c_path = os.path.join(temp_dir, f"proc_c{c_idx}.mp3")
subprocess.run([
"ffmpeg", "-y", "-i", raw_c_path,
"-af", f"silenceremove=1:0:-50dB,atempo={FIXED_SPEED}",
"-threads", "0", proc_c_path
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
processed_chapters.append(proc_c_path)
# Determine the duration of this specific chapter to build precise metadata
dur_str = subprocess.run([
"ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1", proc_c_path
], stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout
try:
duration_ms = int(float(dur_str.decode().strip()) * 1000)
except (ValueError, AttributeError):
duration_ms = 0
metadata_entries.append({
"title": title,
"start": current_time_ms,
"end": current_time_ms + duration_ms
})
current_time_ms += duration_ms
return processed_chapters, metadata_entries, base_name
def final_merge_and_metadata(processed_chapters, metadata_entries, base_name, temp_dir):
# 1. Create FFMETADATA file
meta_path = os.path.join(temp_dir, "metadata.txt")
with open(meta_path, "w", encoding="utf-8") as f:
f.write(";FFMETADATA1\n")
f.write(f"title={base_name}\n\n")
for entry in metadata_entries:
f.write("[CHAPTER]\n")
f.write("TIMEBASE=1/1000\n")
f.write(f"START={entry['start']}\n")
f.write(f"END={entry['end']}\n")
f.write(f"title={entry['title']}\n\n")
# 2. Create concat demuxer file
concat_path = os.path.join(temp_dir, "concat.txt")
with open(concat_path, "w", encoding="utf-8") as f:
for p in processed_chapters:
# FFmpeg requires forward slashes, even on Windows
f.write(f"file '{os.path.abspath(p).replace(chr(92), '/')}'\n")
# 3. Generate outputs
output_mp3 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp3")
output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4")
# Merge to MP3 (Fast Stream Copy)
r_mp3 = subprocess.run([
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
"-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3
], capture_output=True, text=True)
if r_mp3.returncode != 0:
print(f"\u26a0\ufe0f ffmpeg MP3 merge failed for {base_name}:\n{r_mp3.stderr[-800:]}")
# Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility)
r_mp4 = subprocess.run([
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
"-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4
], capture_output=True, text=True)
if r_mp4.returncode != 0:
print(f"\u26a0\ufe0f ffmpeg MP4 merge failed for {base_name}:\n{r_mp4.stderr[-800:]}")
shutil.rmtree(temp_dir)
# Only return files that were actually created
ready = [p for p in [output_mp3, output_mp4] if os.path.isfile(p) and os.path.getsize(p) > 0]
if not ready:
print(f"\u26a0\ufe0f {base_name}: both output files missing after ffmpeg \u2014 bucket sync skipped")
return ready
async def process_single_book(file_path, progress_obj, book_idx, total_books):
temp_dir = tempfile.mkdtemp()
async with download_door:
processed_chapters, metadata_entries, base_name = await asyncio.to_thread(
sync_process_book, file_path, temp_dir, progress_obj, book_idx, total_books
)
if not processed_chapters: return []
async with kitchen_door:
output_files = await asyncio.to_thread(
final_merge_and_metadata, processed_chapters, metadata_entries, base_name, temp_dir
)
# ── Push to HF bucket ──────────────────────────────────────────────────
book_label = f"[Book {book_idx}/{total_books}]"
await asyncio.to_thread(sync_to_hf_bucket, output_files, book_label)
# ──────────────────────────────────────────────────────────────────────
return output_files
async def process_books_holygrail(file_paths, progress=gr.Progress()):
if not file_paths:
yield []
return
if not isinstance(file_paths, list): file_paths = [file_paths]
total = len(file_paths)
tasks = [asyncio.create_task(process_single_book(f, progress, idx+1, total)) for idx, f in enumerate(file_paths)]
finished_files = []
for coro in asyncio.as_completed(tasks):
results = await coro
if results:
finished_files.extend(results)
yield finished_files
if len(finished_files) > 2: # More than 2 files means multiple books processed
zip_path = os.path.join(tempfile.gettempdir(), "AudioNovel_MasterBatch.zip")
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
for f in finished_files: zipf.write(f, os.path.basename(f))
# Sync the master ZIP to HF bucket as well
await asyncio.to_thread(sync_to_hf_bucket, [zip_path], "[Batch ZIP]")
finished_files.insert(0, zip_path)
yield finished_files
with gr.Blocks(theme=gr.themes.Soft(primary_hue="indigo")) as app:
gr.Markdown("# 🎧 AudioNovel Studio (Smooth-Flow Edition)")
with gr.Row():
with gr.Column(scale=1):
file_in = gr.File(label="📂 Upload Books", file_count="multiple")
with gr.Row():
start_btn = gr.Button("🚀 Start Pipeline", variant="primary")
stop_btn = gr.Button("🛑 Stop", variant="stop")
with gr.Column(scale=1):
file_out = gr.File(label="📥 Live Download House (MP3 & MP4)", file_count="multiple", interactive=False)
process_event = start_btn.click(fn=process_books_holygrail, inputs=file_in, outputs=file_out)
stop_btn.click(fn=None, inputs=None, outputs=None, cancels=[process_event])
if __name__ == "__main__":
app.launch()