Krixter commited on
Commit
cfe1f01
Β·
verified Β·
1 Parent(s): 4694030

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +108 -40
app.py CHANGED
@@ -28,9 +28,21 @@ def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool:
28
  shutil.copy2(src, dest)
29
  print(f"πŸ“‚ {book_label} Staged for HF sync: {os.path.basename(src)}")
30
 
 
 
 
 
 
 
31
  try:
32
  result = subprocess.run(
33
- ["hf", "sync", HF_LOCAL_DIR, HF_BUCKET],
 
 
 
 
 
 
34
  capture_output=True, text=True, timeout=300
35
  )
36
  if result.returncode == 0:
@@ -40,7 +52,7 @@ def sync_to_hf_bucket(files: list[str], book_label: str = "") -> bool:
40
  print(f"⚠️ {book_label} HF sync failed:\n{result.stderr.strip()}")
41
  return False
42
  except FileNotFoundError:
43
- print("⚠️ `hf` CLI not found. Install with: pip install huggingface_hub[cli]")
44
  return False
45
  except subprocess.TimeoutExpired:
46
  print(f"⚠️ {book_label} HF sync timed out after 5 minutes.")
@@ -176,6 +188,47 @@ class GoogleTTSEngine:
176
  except: time.sleep(0.5)
177
  return None
178
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
179
  def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
180
  text_file = extract_local_text(file_path, temp_dir)
181
  if not text_file: return None, None, None
@@ -185,46 +238,52 @@ def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
185
  with open(text_file, 'r', encoding='utf-8') as f:
186
  full_text = f.read()
187
 
188
- # 1. PARSE CHAPTERS via Regex with SMART FILTERING
189
- # The [^\n]* captures the REST OF THE LINE so we get names like "Chapter 1: The Awakening"
190
- splits = re.split(r'(?i)(\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*|\b\d+\s*:(?!\d)[^\n]*)', full_text)
191
  chapters = []
192
-
 
193
  if splits[0].strip():
194
  chapters.append(["Intro", splits[0].replace('\n', ' ').strip()])
195
-
196
  for i in range(1, len(splits), 2):
197
  title = splits[i].strip()
198
- body = splits[i+1].strip() if i+1 < len(splits) else ""
199
-
200
- # SAFEGUARD: If a newline was missing and it grabbed a whole paragraph as a title
201
  if len(title) > 100:
202
- body = title[100:] + " " + body
203
  title = title[:100] + "..."
204
 
205
- # Extract just the number for comparison (e.g., "CHAPTER 1: The Dark" -> "1")
206
- num_match = re.search(r'\d+', title)
207
- current_num = num_match.group() if num_match else title
208
 
209
- if chapters:
210
- prev_title = chapters[-1][0]
211
- prev_num_match = re.search(r'\d+', prev_title)
212
- prev_num = prev_num_match.group() if prev_num_match else prev_title
213
-
214
- # EDGE CASE 1: Double Headings
215
- if current_num == prev_num:
216
- chapters[-1][1] += f" {body}".replace('\n', ' ')
217
- continue
218
-
219
- # EDGE CASE 2: Table of Contents or tiny chapters (< 150 characters)
220
- if len(chapters[-1][1]) < 150:
221
  chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
222
- chapters[-1][0] = title # Rename to the current valid title
223
- continue
 
 
 
 
 
 
 
224
 
225
- # Convert the actual TTS body back to single-line text so the TTS engine doesn't stutter
226
- clean_text_for_tts = f"{title}. {body}".replace('\n', ' ')
227
- chapters.append([title, clean_text_for_tts])
 
 
 
 
 
 
 
 
228
 
229
  if not chapters:
230
  chapters = [["Full Book", full_text.replace('\n', ' ')]]
@@ -334,19 +393,28 @@ def final_merge_and_metadata(processed_chapters, metadata_entries, base_name, te
334
  output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4")
335
 
336
  # Merge to MP3 (Fast Stream Copy)
337
- subprocess.run([
338
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
339
  "-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3
340
- ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
341
-
 
 
342
  # Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility)
343
- subprocess.run([
344
- "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
345
  "-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4
346
- ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
347
-
 
 
348
  shutil.rmtree(temp_dir)
349
- return [output_mp3, output_mp4]
 
 
 
 
 
350
 
351
  async def process_single_book(file_path, progress_obj, book_idx, total_books):
352
  temp_dir = tempfile.mkdtemp()
 
28
  shutil.copy2(src, dest)
29
  print(f"πŸ“‚ {book_label} Staged for HF sync: {os.path.basename(src)}")
30
 
31
+ # HF_BUCKET is "hf://buckets/Krixter/storage"
32
+ # Parse out repo_id and subfolder from the hf:// URI
33
+ bucket_path = HF_BUCKET.replace("hf://buckets/", "") # β†’ "Krixter/storage"
34
+ repo_id = bucket_path.split("/")[0] # β†’ "Krixter"
35
+ path_in_repo = "/".join(bucket_path.split("/")[1:]) # β†’ "storage"
36
+
37
  try:
38
  result = subprocess.run(
39
+ [
40
+ "huggingface-cli", "upload",
41
+ "--repo-type", "dataset",
42
+ repo_id,
43
+ HF_LOCAL_DIR, # local folder to upload
44
+ path_in_repo, # destination path inside the repo
45
+ ],
46
  capture_output=True, text=True, timeout=300
47
  )
48
  if result.returncode == 0:
 
52
  print(f"⚠️ {book_label} HF sync failed:\n{result.stderr.strip()}")
53
  return False
54
  except FileNotFoundError:
55
+ print("⚠️ `huggingface-cli` not found. Install with: pip install huggingface_hub[cli]")
56
  return False
57
  except subprocess.TimeoutExpired:
58
  print(f"⚠️ {book_label} HF sync timed out after 5 minutes.")
 
188
  except: time.sleep(0.5)
189
  return None
190
 
191
+ def _detect_and_split(full_text):
192
+ """
193
+ Tries chapter-heading formats in priority order and returns re.split() results
194
+ for the first format that produces a mostly-ascending sequence of β‰₯ 2 chapters.
195
+
196
+ Priority order (highest β†’ lowest):
197
+ 1. "Chapter N" / "Ch. N" β€” explicit word, most authoritative
198
+ 2. "N: Title" β€” bare number-colon at line start
199
+
200
+ 'Section N' is intentionally excluded: if a book only uses that scheme the
201
+ function returns a single-element list so the caller falls back to Full Book.
202
+ Numbers must be in ascending order for the format to be accepted; a format
203
+ where the numbers jump around (e.g. a ToC bleed) is skipped in favour of the
204
+ next candidate.
205
+ """
206
+ FORMATS = [
207
+ # P1 – word "chapter" / "ch." anywhere on its own line
208
+ (r'(?im)(^[ \t]*\b(?:chapter|ch\.?)\s*[-:]?\s*\d+\b[^\n]*)', 'chapter_word'),
209
+ # P2 – bare "3: Title" at line start (not timestamps like 1:30)
210
+ (r'(?m)(^[ \t]*\d+\s*:(?!\d)[^\n]*)', 'number_colon'),
211
+ ]
212
+
213
+ for pattern, _name in FORMATS:
214
+ candidates = re.findall(pattern, full_text)
215
+ if len(candidates) < 2:
216
+ continue
217
+ nums = []
218
+ for c in candidates:
219
+ m = re.search(r'\d+', c)
220
+ if m:
221
+ nums.append(int(m.group()))
222
+ if len(nums) < 2:
223
+ continue
224
+ ascending = sum(1 for j in range(1, len(nums)) if nums[j] > nums[j - 1])
225
+ if ascending / (len(nums) - 1) >= 0.6: # β‰₯ 60 % of transitions go up β†’ valid
226
+ return re.split(pattern, full_text)
227
+
228
+ # No usable format found β†’ treat whole text as one block
229
+ return [full_text]
230
+
231
+
232
  def sync_process_book(file_path, temp_dir, progress_obj, book_idx, total_books):
233
  text_file = extract_local_text(file_path, temp_dir)
234
  if not text_file: return None, None, None
 
238
  with open(text_file, 'r', encoding='utf-8') as f:
239
  full_text = f.read()
240
 
241
+ # 1. PARSE CHAPTERS β€” detect format first, then split with that format only
242
+ splits = _detect_and_split(full_text)
 
243
  chapters = []
244
+ prev_num = 0 # highest chapter-number *accepted* so far (TOC merges don't count)
245
+
246
  if splits[0].strip():
247
  chapters.append(["Intro", splits[0].replace('\n', ' ').strip()])
248
+
249
  for i in range(1, len(splits), 2):
250
  title = splits[i].strip()
251
+ body = splits[i + 1].strip() if i + 1 < len(splits) else ""
252
+
253
+ # SAFEGUARD: title grabbed a whole paragraph (newline missing)
254
  if len(title) > 100:
255
+ body = title[100:] + " " + body
256
  title = title[:100] + "..."
257
 
258
+ num_match = re.search(r'\d+', title)
259
+ current_num = int(num_match.group()) if num_match else None
 
260
 
261
+ # ── OUT-OF-ORDER: number went backward β†’ just a back-reference/mention ──
262
+ # (only enforced once at least one real chapter has been accepted)
263
+ if current_num is not None and prev_num > 0 and current_num < prev_num:
264
+ if chapters:
 
 
 
 
 
 
 
 
265
  chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
266
+ continue
267
+
268
+ # ── DOUBLE HEADING: same number as last accepted chapter ──
269
+ # (covers "TOC entry" followed by "real chapter" with same number)
270
+ if current_num is not None and chapters:
271
+ last_m = re.search(r'\d+', chapters[-1][0])
272
+ if last_m and int(last_m.group()) == current_num:
273
+ chapters[-1][1] += f" {body}".replace('\n', ' ')
274
+ continue # prev_num unchanged β€” body absorbed, not a new chapter
275
 
276
+ # ── TOC / TINY: previous chapter body still very short ──
277
+ # Merge in and rename, but do NOT update prev_num (these are TOC lines)
278
+ if chapters and len(chapters[-1][1]) < 150:
279
+ chapters[-1][1] += f" {title}. {body}".replace('\n', ' ')
280
+ chapters[-1][0] = title
281
+ continue
282
+
283
+ # ── Accept as a real chapter ──
284
+ chapters.append([title, f"{title}. {body}".replace('\n', ' ')])
285
+ if current_num is not None:
286
+ prev_num = current_num
287
 
288
  if not chapters:
289
  chapters = [["Full Book", full_text.replace('\n', ' ')]]
 
393
  output_mp4 = os.path.join(tempfile.gettempdir(), f"{base_name}_Audio.mp4")
394
 
395
  # Merge to MP3 (Fast Stream Copy)
396
+ r_mp3 = subprocess.run([
397
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
398
  "-i", meta_path, "-map_metadata", "1", "-c", "copy", output_mp3
399
+ ], capture_output=True, text=True)
400
+ if r_mp3.returncode != 0:
401
+ print(f"\u26a0\ufe0f ffmpeg MP3 merge failed for {base_name}:\n{r_mp3.stderr[-800:]}")
402
+
403
  # Merge to MP4 (Converts to AAC for universal MP4/M4B chapter compatibility)
404
+ r_mp4 = subprocess.run([
405
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", concat_path,
406
  "-i", meta_path, "-map_metadata", "1", "-c:a", "aac", "-b:a", "48k", output_mp4
407
+ ], capture_output=True, text=True)
408
+ if r_mp4.returncode != 0:
409
+ print(f"\u26a0\ufe0f ffmpeg MP4 merge failed for {base_name}:\n{r_mp4.stderr[-800:]}")
410
+
411
  shutil.rmtree(temp_dir)
412
+
413
+ # Only return files that were actually created
414
+ ready = [p for p in [output_mp3, output_mp4] if os.path.isfile(p) and os.path.getsize(p) > 0]
415
+ if not ready:
416
+ print(f"\u26a0\ufe0f {base_name}: both output files missing after ffmpeg \u2014 bucket sync skipped")
417
+ return ready
418
 
419
  async def process_single_book(file_path, progress_obj, book_idx, total_books):
420
  temp_dir = tempfile.mkdtemp()