Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -28,9 +28,21 @@ def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool:
|
|
| 28 |
shutil.copy2(src, dest)
|
| 29 |
print(f"π {book_label} Staged for HF sync: {os.path.basename(src)}")
|
| 30 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
try:
|
| 32 |
result = subprocess.run(
|
| 33 |
-
[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
capture_output=True, text=True, timeout=300
|
| 35 |
)
|
| 36 |
if result.returncode == 0:
|
|
@@ -40,7 +52,7 @@ def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool:
|
|
| 40 |
print(f"β οΈ {book_label} HF sync failed:\n{result.stderr.strip()}")
|
| 41 |
return False
|
| 42 |
except FileNotFoundError:
|
| 43 |
-
print("β οΈ `
|
| 44 |
return False
|
| 45 |
except subprocess.TimeoutExpired:
|
| 46 |
print(f"β οΈ {book_label} HF sync timed out after 5 minutes.")
|
|
@@ -176,6 +188,47 @@ class GoogleTTSEngine:
|
|
| 176 |
except: time.sleep(0.5)
|
| 177 |
return None
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
|
| 180 |
text_file = extract_local_text(file_path, temp_dir)
|
| 181 |
if not text_file: return None, None, None
|
|
@@ -185,46 +238,52 @@ def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
|
|
| 185 |
with open(text_file, 'r', encoding='utf-8') as f:
|
| 186 |
full_text = f.read()
|
| 187 |
|
| 188 |
-
# 1. PARSE CHAPTERS
|
| 189 |
-
|
| 190 |
-
splits = re.split(r'(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*|\b\d+\s*:(?!\d)[^\n]*)', full_text)
|
| 191 |
chapters = []
|
| 192 |
-
|
|
|
|
| 193 |
if splits[0].strip():
|
| 194 |
chapters.append(["Intro", splits[0].replace('\n', ' ').strip()])
|
| 195 |
-
|
| 196 |
for i in range(1, len(splits), 2):
|
| 197 |
title = splits[i].strip()
|
| 198 |
-
body
|
| 199 |
-
|
| 200 |
-
# SAFEGUARD:
|
| 201 |
if len(title) > 100:
|
| 202 |
-
body
|
| 203 |
title = title[:100] + "..."
|
| 204 |
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
current_num = num_match.group() if num_match else title
|
| 208 |
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
# EDGE CASE 1: Double Headings
|
| 215 |
-
if current_num == prev_num:
|
| 216 |
-
chapters[-1][1] += f" {body}".replace('\n', ' ')
|
| 217 |
-
continue
|
| 218 |
-
|
| 219 |
-
# EDGE CASE 2: Table of Contents or tiny chapters (< 150 characters)
|
| 220 |
-
if len(chapters[-1][1]) < 150:
|
| 221 |
chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
|
| 222 |
-
|
| 223 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
|
| 225 |
-
#
|
| 226 |
-
|
| 227 |
-
chapters
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
|
| 229 |
if not chapters:
|
| 230 |
chapters = [["Full Book", full_text.replace('\n', ' ')]]
|
|
@@ -334,19 +393,28 @@ def final_merge_and_metadata(processed_chapters, metadata_entries, base_name, te
|
|
| 334 |
output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4")
|
| 335 |
|
| 336 |
# Merge to MP3 (Fast Stream Copy)
|
| 337 |
-
subprocess.run([
|
| 338 |
-
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
|
| 339 |
"-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3
|
| 340 |
-
],
|
| 341 |
-
|
|
|
|
|
|
|
| 342 |
# Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility)
|
| 343 |
-
subprocess.run([
|
| 344 |
-
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
|
| 345 |
"-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4
|
| 346 |
-
],
|
| 347 |
-
|
|
|
|
|
|
|
| 348 |
shutil.rmtree(temp_dir)
|
| 349 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 350 |
|
| 351 |
async def process_single_book(file_path, progress_obj, book_idx, total_books):
|
| 352 |
temp_dir = tempfile.mkdtemp()
|
|
|
|
| 28 |
shutil.copy2(src, dest)
|
| 29 |
print(f"π {book_label} Staged for HF sync: {os.path.basename(src)}")
|
| 30 |
|
| 31 |
+
# HF_BUCKET is "hf://buckets/Krixter/storage"
|
| 32 |
+
# Parse out repo_id and subfolder from the hf:// URI
|
| 33 |
+
bucket_path = HF_BUCKET.replace("hf://buckets/", "") # β "Krixter/storage"
|
| 34 |
+
repo_id = bucket_path.split("/")[0] # β "Krixter"
|
| 35 |
+
path_in_repo = "/".join(bucket_path.split("/")[1:]) # β "storage"
|
| 36 |
+
|
| 37 |
try:
|
| 38 |
result = subprocess.run(
|
| 39 |
+
[
|
| 40 |
+
"huggingface-cli", "upload",
|
| 41 |
+
"--repo-type", "dataset",
|
| 42 |
+
repo_id,
|
| 43 |
+
HF_LOCAL_DIR, # local folder to upload
|
| 44 |
+
path_in_repo, # destination path inside the repo
|
| 45 |
+
],
|
| 46 |
capture_output=True, text=True, timeout=300
|
| 47 |
)
|
| 48 |
if result.returncode == 0:
|
|
|
|
| 52 |
print(f"β οΈ {book_label} HF sync failed:\n{result.stderr.strip()}")
|
| 53 |
return False
|
| 54 |
except FileNotFoundError:
|
| 55 |
+
print("β οΈ `huggingface-cli` not found. Install with: pip install huggingface_hub[cli]")
|
| 56 |
return False
|
| 57 |
except subprocess.TimeoutExpired:
|
| 58 |
print(f"β οΈ {book_label} HF sync timed out after 5 minutes.")
|
|
|
|
| 188 |
except: time.sleep(0.5)
|
| 189 |
return None
|
| 190 |
|
| 191 |
+
def _detect_and_split(full_text):
|
| 192 |
+
"""
|
| 193 |
+
Tries chapter-heading formats in priority order and returns re.split() results
|
| 194 |
+
for the first format that produces a mostly-ascending sequence of β₯ 2 chapters.
|
| 195 |
+
|
| 196 |
+
Priority order (highest β lowest):
|
| 197 |
+
1. "Chapter N" / "Ch. N" β explicit word, most authoritative
|
| 198 |
+
2. "N: Title" β bare number-colon at line start
|
| 199 |
+
|
| 200 |
+
'Section N' is intentionally excluded: if a book only uses that scheme the
|
| 201 |
+
function returns a single-element list so the caller falls back to Full Book.
|
| 202 |
+
Numbers must be in ascending order for the format to be accepted; a format
|
| 203 |
+
where the numbers jump around (e.g. a ToC bleed) is skipped in favour of the
|
| 204 |
+
next candidate.
|
| 205 |
+
"""
|
| 206 |
+
FORMATS = [
|
| 207 |
+
# P1 β word "chapter" / "ch." anywhere on its own line
|
| 208 |
+
(r'(?im)(^[ \t]*\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*)', 'chapter_word'),
|
| 209 |
+
# P2 β bare "3: Title" at line start (not timestamps like 1:30)
|
| 210 |
+
(r'(?m)(^[ \t]*\d+\s*:(?!\d)[^\n]*)', 'number_colon'),
|
| 211 |
+
]
|
| 212 |
+
|
| 213 |
+
for pattern, _name in FORMATS:
|
| 214 |
+
candidates = re.findall(pattern, full_text)
|
| 215 |
+
if len(candidates) < 2:
|
| 216 |
+
continue
|
| 217 |
+
nums = []
|
| 218 |
+
for c in candidates:
|
| 219 |
+
m = re.search(r'\d+', c)
|
| 220 |
+
if m:
|
| 221 |
+
nums.append(int(m.group()))
|
| 222 |
+
if len(nums) < 2:
|
| 223 |
+
continue
|
| 224 |
+
ascending = sum(1 for j in range(1, len(nums)) if nums[j] > nums[j - 1])
|
| 225 |
+
if ascending / (len(nums) - 1) >= 0.6: # β₯ 60 % of transitions go up β valid
|
| 226 |
+
return re.split(pattern, full_text)
|
| 227 |
+
|
| 228 |
+
# No usable format found β treat whole text as one block
|
| 229 |
+
return [full_text]
|
| 230 |
+
|
| 231 |
+
|
| 232 |
def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
|
| 233 |
text_file = extract_local_text(file_path, temp_dir)
|
| 234 |
if not text_file: return None, None, None
|
|
|
|
| 238 |
with open(text_file, 'r', encoding='utf-8') as f:
|
| 239 |
full_text = f.read()
|
| 240 |
|
| 241 |
+
# 1. PARSE CHAPTERS β detect format first, then split with that format only
|
| 242 |
+
splits = _detect_and_split(full_text)
|
|
|
|
| 243 |
chapters = []
|
| 244 |
+
prev_num = 0 # highest chapter-number *accepted* so far (TOC merges don't count)
|
| 245 |
+
|
| 246 |
if splits[0].strip():
|
| 247 |
chapters.append(["Intro", splits[0].replace('\n', ' ').strip()])
|
| 248 |
+
|
| 249 |
for i in range(1, len(splits), 2):
|
| 250 |
title = splits[i].strip()
|
| 251 |
+
body = splits[i + 1].strip() if i + 1 < len(splits) else ""
|
| 252 |
+
|
| 253 |
+
# SAFEGUARD: title grabbed a whole paragraph (newline missing)
|
| 254 |
if len(title) > 100:
|
| 255 |
+
body = title[100:] + " " + body
|
| 256 |
title = title[:100] + "..."
|
| 257 |
|
| 258 |
+
num_match = re.search(r'\d+', title)
|
| 259 |
+
current_num = int(num_match.group()) if num_match else None
|
|
|
|
| 260 |
|
| 261 |
+
# ββ OUT-OF-ORDER: number went backward β just a back-reference/mention ββ
|
| 262 |
+
# (only enforced once at least one real chapter has been accepted)
|
| 263 |
+
if current_num is not None and prev_num > 0 and current_num < prev_num:
|
| 264 |
+
if chapters:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
|
| 266 |
+
continue
|
| 267 |
+
|
| 268 |
+
# ββ DOUBLE HEADING: same number as last accepted chapter ββ
|
| 269 |
+
# (covers "TOC entry" followed by "real chapter" with same number)
|
| 270 |
+
if current_num is not None and chapters:
|
| 271 |
+
last_m = re.search(r'\d+', chapters[-1][0])
|
| 272 |
+
if last_m and int(last_m.group()) == current_num:
|
| 273 |
+
chapters[-1][1] += f" {body}".replace('\n', ' ')
|
| 274 |
+
continue # prev_num unchanged β body absorbed, not a new chapter
|
| 275 |
|
| 276 |
+
# ββ TOC / TINY: previous chapter body still very short ββ
|
| 277 |
+
# Merge in and rename, but do NOT update prev_num (these are TOC lines)
|
| 278 |
+
if chapters and len(chapters[-1][1]) < 150:
|
| 279 |
+
chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
|
| 280 |
+
chapters[-1][0] = title
|
| 281 |
+
continue
|
| 282 |
+
|
| 283 |
+
# ββ Accept as a real chapter ββ
|
| 284 |
+
chapters.append([title, f"{title}. {body}".replace('\n', ' ')])
|
| 285 |
+
if current_num is not None:
|
| 286 |
+
prev_num = current_num
|
| 287 |
|
| 288 |
if not chapters:
|
| 289 |
chapters = [["Full Book", full_text.replace('\n', ' ')]]
|
|
|
|
| 393 |
output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4")
|
| 394 |
|
| 395 |
# Merge to MP3 (Fast Stream Copy)
|
| 396 |
+
r_mp3 = subprocess.run([
|
| 397 |
+
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
|
| 398 |
"-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3
|
| 399 |
+
], capture_output=True, text=True)
|
| 400 |
+
if r_mp3.returncode != 0:
|
| 401 |
+
print(f"\u26a0\ufe0f ffmpeg MP3 merge failed for {base_name}:\n{r_mp3.stderr[-800:]}")
|
| 402 |
+
|
| 403 |
# Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility)
|
| 404 |
+
r_mp4 = subprocess.run([
|
| 405 |
+
"ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
|
| 406 |
"-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4
|
| 407 |
+
], capture_output=True, text=True)
|
| 408 |
+
if r_mp4.returncode != 0:
|
| 409 |
+
print(f"\u26a0\ufe0f ffmpeg MP4 merge failed for {base_name}:\n{r_mp4.stderr[-800:]}")
|
| 410 |
+
|
| 411 |
shutil.rmtree(temp_dir)
|
| 412 |
+
|
| 413 |
+
# Only return files that were actually created
|
| 414 |
+
ready = [p for p in [output_mp3, output_mp4] if os.path.isfile(p) and os.path.getsize(p) > 0]
|
| 415 |
+
if not ready:
|
| 416 |
+
print(f"\u26a0\ufe0f {base_name}: both output files missing after ffmpeg \u2014 bucket sync skipped")
|
| 417 |
+
return ready
|
| 418 |
|
| 419 |
async def process_single_book(file_path, progress_obj, book_idx, total_books):
|
| 420 |
temp_dir = tempfile.mkdtemp()
|