Adeel020 commited on
Commit
7e4e7bb
Β·
verified Β·
1 Parent(s): 8ae7f9f

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +56 -99
app.py CHANGED
@@ -16,32 +16,8 @@ import subprocess
16
  import urllib.request
17
  from pathlib import Path
18
 
19
- # Force Gradio temp dir configuration
20
- os.environ["GRADIO_TEMP_DIR"] = "/tmp/gradio"
21
- Path("/tmp/gradio").mkdir(parents=True, exist_ok=True)
22
-
23
  import gradio as gr
24
 
25
- # ── Monkeypatch Gradio Client bug in Python 3.13 / Pydantic v2 ───────────────
26
- try:
27
- import gradio_client.utils
28
- orig_get_type = gradio_client.utils.get_type
29
- def patched_get_type(schema):
30
- if isinstance(schema, bool):
31
- return "any"
32
- return orig_get_type(schema)
33
- gradio_client.utils.get_type = patched_get_type
34
-
35
- orig_json_schema_to_python_type = gradio_client.utils._json_schema_to_python_type
36
- def patched_json_schema_to_python_type(schema, defs=None):
37
- if isinstance(schema, bool):
38
- return "any"
39
- return orig_json_schema_to_python_type(schema, defs)
40
- gradio_client.utils._json_schema_to_python_type = patched_json_schema_to_python_type
41
- print("[patch] Gradio Client JSON schema parser patched successfully.")
42
- except Exception as patch_err:
43
- print(f"[patch] Failed to apply Gradio Client patch: {patch_err}")
44
-
45
  # ── Paths ─────────────────────────────────────────────────────────────────────
46
  MODELS_DIR = Path("/tmp/voicekit_models")
47
  WAV2LIP_DIR = MODELS_DIR / "Wav2Lip"
@@ -56,7 +32,7 @@ CKPT_URLS = [
56
  "https://huggingface.co/numz/wav2lip_studio/resolve/main/Wav2lip/wav2lip_gan.pth",
57
  "https://huggingface.co/camenduru/wav2lip/resolve/main/wav2lip_gan.pth",
58
  ]
59
- FACE_DET_URL = "https://huggingface.co/camenduru/Wav2Lip/resolve/main/checkpoints/s3fd-619a316812.pth"
60
 
61
 
62
  # ── One-time Model Bootstrap ──────────────────────────────────────────────────
@@ -107,64 +83,60 @@ def bootstrap():
107
  # Face detector
108
  _download(FACE_DET_URL, FACE_DET, "S3FD face detector (~85MB)")
109
 
110
- # Patch audio.py for newer librosa versions
111
- audio_py = WAV2LIP_DIR / "audio.py"
112
- if audio_py.exists():
113
- content = audio_py.read_text(encoding="utf-8", errors="ignore")
114
- if "hp.sample_rate, hp.n_fft" in content:
115
- content = content.replace(
116
- "hp.sample_rate, hp.n_fft",
117
- "sr=hp.sample_rate, n_fft=hp.n_fft"
118
- )
119
- audio_py.write_text(content, encoding="utf-8")
120
- print("[patch] Patched audio.py for librosa filters.mel signature")
121
-
122
  print("[ready] All models ready")
123
 
124
 
125
- # Setup variables (models will be lazy-loaded on first generation request to prevent startup timeout)
126
- SETUP_OK = True
127
- SETUP_ERR = ""
128
- MODELS_READY = False
 
 
 
 
 
129
 
130
 
131
  # ── Voice Generation ──────────────────────────────────────────────────────────
132
 
133
  def generate_voice(script: str, voice_sample_path: str) -> str:
134
- """Generate speech using the public F5-TTS API (zero-shot cloning)."""
135
- import os
136
- from gradio_client import Client, handle_file
137
 
138
  out_path = str(OUTPUT_DIR / f"voice_{int(time.time())}.wav")
139
- print(f"[tts] Calling Public F5-TTS API...")
140
-
141
  try:
142
- client = Client("abidlabs/E2-F5-TTS")
143
- result = client.predict(
144
- ref_audio=handle_file(voice_sample_path),
145
- ref_text="",
146
- gen_text=script,
147
- remove_silence=False,
148
- api_name="/predict"
149
- )
150
- import shutil
151
- shutil.copy2(result[0], out_path)
 
 
 
 
 
 
152
  return out_path
153
  except Exception as e:
154
- print(f"[tts] F5-TTS failed: {e}")
155
- raise RuntimeError(f"F5-TTS API error: {e}")
156
 
157
 
158
- def generate_voice_edgetts(script: str, accent: str = "us") -> str:
159
  """Fallback: edge-tts generic voice."""
160
  import asyncio
161
  import edge_tts
162
  out_path = str(OUTPUT_DIR / f"voice_generic_{int(time.time())}.mp3")
163
 
164
- edge_voice = "en-GB-SoniaNeural" if accent == "uk" else "en-US-AriaNeural"
165
-
166
  async def _gen():
167
- c = edge_tts.Communicate(script, edge_voice)
168
  await c.save(out_path)
169
 
170
  asyncio.run(_gen())
@@ -175,7 +147,7 @@ def generate_voice_edgetts(script: str, accent: str = "us") -> str:
175
 
176
  def run_wav2lip(video_path: str, audio_path: str) -> str:
177
  """Run Wav2Lip inference and return path to output video."""
178
- out_path = str(OUTPUT_DIR / f"voicekit_{int(time.time())}.mp4")
179
  tmp_out = str(WAV2LIP_DIR / "results" / "result_voice.mp4")
180
  os.makedirs(os.path.dirname(tmp_out), exist_ok=True)
181
 
@@ -186,7 +158,6 @@ def run_wav2lip(video_path: str, audio_path: str) -> str:
186
  "--face", video_path,
187
  "--audio", audio_path,
188
  "--outfile", tmp_out,
189
- "--resize_factor", "2",
190
  "--nosmooth",
191
  ]
192
  print(f"[wav2lip] Running inference...")
@@ -201,19 +172,8 @@ def run_wav2lip(video_path: str, audio_path: str) -> str:
201
 
202
  # ── Main Pipeline ─────────────────────────────────────────────────────────────
203
 
204
- def voicekit_generate(video_file, voice_file, script, use_voice_clone, accent, progress=gr.Progress()):
205
  """Main VoiceKit pipeline called by Gradio."""
206
- global MODELS_READY, SETUP_OK, SETUP_ERR
207
-
208
- if not MODELS_READY:
209
- progress(0.01, desc="Downloading Wav2Lip models & checking configurations (first-time setup)...")
210
- try:
211
- bootstrap()
212
- MODELS_READY = True
213
- except Exception as e:
214
- SETUP_OK = False
215
- SETUP_ERR = str(e)
216
- raise gr.Error(f"Setup failed: {e}")
217
 
218
  if not SETUP_OK:
219
  raise gr.Error(f"Setup failed: {SETUP_ERR}")
@@ -226,17 +186,17 @@ def voicekit_generate(video_file, voice_file, script, use_voice_clone, accent, p
226
 
227
  # Step 1: Generate audio
228
  if use_voice_clone and voice_file is not None:
229
- progress(0.15, desc="Cloning your exact voice with F5-TTS API...")
230
  try:
231
  audio_path = generate_voice(script, voice_file)
232
- voice_note = "Voice cloned successfully"
233
  except Exception as e:
234
- progress(0.15, desc=f"Voice cloning failed ({e}), using generic voice...")
235
- audio_path = generate_voice_edgetts(script, accent)
236
- voice_note = f"Used generic voice (cloning failed: {e})"
237
  else:
238
  progress(0.2, desc="Generating audio with generic voice...")
239
- audio_path = generate_voice_edgetts(script, accent)
240
  voice_note = "Used generic voice (no voice sample provided)"
241
 
242
  progress(0.5, desc="Running Wav2Lip lip-sync (1-5 min on CPU, ~1 min on GPU)...")
@@ -275,7 +235,11 @@ INSTRUCTIONS = """
275
  - The face should be **clearly visible** throughout the video
276
  """
277
 
278
- STATUS_MSG = "VoiceKit API server running. Models will load automatically on first request."
 
 
 
 
279
 
280
  css = """
281
  body { font-family: 'Inter', sans-serif; }
@@ -298,9 +262,9 @@ with gr.Blocks(
298
  with gr.Row():
299
  with gr.Column(scale=3):
300
  gr.Markdown("### Step 1 β€” Upload Your Face Video")
301
- video_input = gr.File(
302
  label="Face Video (MP4 / MKV)",
303
- file_types=[".mp4", ".mkv"],
304
  )
305
 
306
  gr.Markdown("### Step 2 β€” Upload Your Voice Sample")
@@ -314,19 +278,13 @@ with gr.Blocks(
314
  value=True,
315
  )
316
 
317
- gr.Markdown("### Step 3 β€” Type Your Script & Options")
318
  script_input = gr.Textbox(
319
  label="Script",
320
  placeholder="Hey [Name], I've been watching what you're doing and honestly there's a way to 10x your outreach using AI. Worth 2 minutes of your time?",
321
  lines=6,
322
  max_lines=20,
323
  )
324
-
325
- accent_input = gr.Radio(
326
- ["us", "uk"],
327
- label="Accent (used for generic voice)",
328
- value="us"
329
- )
330
 
331
  generate_btn = gr.Button(
332
  "Generate Lip-Synced Video",
@@ -336,7 +294,7 @@ with gr.Blocks(
336
 
337
  with gr.Column(scale=2):
338
  gr.Markdown("### Output")
339
- video_output = gr.File(label="Your Lip-Synced Video")
340
  status_text = gr.Textbox(label="Status", interactive=False)
341
 
342
  gr.Markdown(INSTRUCTIONS)
@@ -347,18 +305,17 @@ with gr.Blocks(
347
  )
348
 
349
  generate_btn.click(
350
- fn=voicekit_generate,
351
- inputs=[video_input, voice_input, script_input, use_clone, accent_input],
352
  outputs=[video_output, status_text],
353
- api_name="lipkit_generate",
354
  )
355
 
356
  gr.Examples(
357
  examples=[
358
- [None, None, "Hey, I've been keeping an eye on your content and honestly β€” the engagement you're getting is impressive. I built an AI tool that could help you turn one video into 50 personalized ones automatically. Want to see how it works?", True, "us"],
359
- [None, None, "What's up! I help fintech founders like you turn one video into a hundred personalized outreach clips using AI. No editing skills needed, no extra time. Want me to show you a quick demo?", True, "uk"],
360
  ],
361
- inputs=[video_input, voice_input, script_input, use_clone, accent_input],
362
  label="Script Examples",
363
  )
364
 
 
16
  import urllib.request
17
  from pathlib import Path
18
 
 
 
 
 
19
  import gradio as gr
20
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  # ── Paths ─────────────────────────────────────────────────────────────────────
22
  MODELS_DIR = Path("/tmp/voicekit_models")
23
  WAV2LIP_DIR = MODELS_DIR / "Wav2Lip"
 
32
  "https://huggingface.co/numz/wav2lip_studio/resolve/main/Wav2lip/wav2lip_gan.pth",
33
  "https://huggingface.co/camenduru/wav2lip/resolve/main/wav2lip_gan.pth",
34
  ]
35
+ FACE_DET_URL = "https://huggingface.co/camenduru/wav2lip/resolve/main/face_detection/s3fd-619a316812.pth"
36
 
37
 
38
  # ── One-time Model Bootstrap ──────────────────────────────────────────────────
 
83
  # Face detector
84
  _download(FACE_DET_URL, FACE_DET, "S3FD face detector (~85MB)")
85
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  print("[ready] All models ready")
87
 
88
 
89
+ # Run bootstrap at startup
90
+ try:
91
+ bootstrap()
92
+ SETUP_OK = True
93
+ SETUP_ERR = ""
94
+ except Exception as e:
95
+ SETUP_OK = False
96
+ SETUP_ERR = str(e)
97
+ print(f"[ERROR] Bootstrap failed: {e}")
98
 
99
 
100
  # ── Voice Generation ──────────────────────────────────────────────────────────
101
 
102
  def generate_voice(script: str, voice_sample_path: str) -> str:
103
+ """Generate speech using Kokoro TTS (Python 3.14 compatible)."""
104
+ import numpy as np
105
+ import soundfile as sf
106
 
107
  out_path = str(OUTPUT_DIR / f"voice_{int(time.time())}.wav")
108
+
 
109
  try:
110
+ from kokoro_onnx import Kokoro
111
+ print("[tts] Using Kokoro ONNX")
112
+ kokoro = Kokoro("kokoro-v1.0.onnx", "voices-v1.0.bin")
113
+ samples, sr = kokoro.create(script, voice="af_heart", speed=1.0, lang="en-us")
114
+ sf.write(out_path, samples, sr)
115
+ return out_path
116
+ except Exception:
117
+ pass
118
+
119
+ try:
120
+ import kokoro as k
121
+ print("[tts] Using Kokoro pipeline")
122
+ pipeline = k.KPipeline(lang_code="en")
123
+ parts = [audio for _, _, audio in pipeline(script, voice="af_heart", speed=1.0)]
124
+ combined = np.concatenate(parts) if len(parts) > 1 else parts[0]
125
+ sf.write(out_path, combined, 24000)
126
  return out_path
127
  except Exception as e:
128
+ print(f"[tts] Kokoro failed: {e}, using edge-tts")
129
+ return generate_voice_edgetts(script)
130
 
131
 
132
+ def generate_voice_edgetts(script: str) -> str:
133
  """Fallback: edge-tts generic voice."""
134
  import asyncio
135
  import edge_tts
136
  out_path = str(OUTPUT_DIR / f"voice_generic_{int(time.time())}.mp3")
137
 
 
 
138
  async def _gen():
139
+ c = edge_tts.Communicate(script, "en-US-GuyNeural")
140
  await c.save(out_path)
141
 
142
  asyncio.run(_gen())
 
147
 
148
  def run_wav2lip(video_path: str, audio_path: str) -> str:
149
  """Run Wav2Lip inference and return path to output video."""
150
+ out_path = str(OUTPUT_DIR / f"lipkit_{int(time.time())}.mp4")
151
  tmp_out = str(WAV2LIP_DIR / "results" / "result_voice.mp4")
152
  os.makedirs(os.path.dirname(tmp_out), exist_ok=True)
153
 
 
158
  "--face", video_path,
159
  "--audio", audio_path,
160
  "--outfile", tmp_out,
 
161
  "--nosmooth",
162
  ]
163
  print(f"[wav2lip] Running inference...")
 
172
 
173
  # ── Main Pipeline ─────────────────────────────────────────────────────────────
174
 
175
+ def lipkit_generate(video_file, voice_file, script, use_voice_clone, progress=gr.Progress()):
176
  """Main VoiceKit pipeline called by Gradio."""
 
 
 
 
 
 
 
 
 
 
 
177
 
178
  if not SETUP_OK:
179
  raise gr.Error(f"Setup failed: {SETUP_ERR}")
 
186
 
187
  # Step 1: Generate audio
188
  if use_voice_clone and voice_file is not None:
189
+ progress(0.15, desc="Generating audio with Kokoro...")
190
  try:
191
  audio_path = generate_voice(script, voice_file)
192
+ voice_note = "Voice generated successfully"
193
  except Exception as e:
194
+ progress(0.15, desc=f"Voice generation failed ({e}), using generic voice...")
195
+ audio_path = generate_voice_edgetts(script)
196
+ voice_note = f"Used generic voice (generation failed: {e})"
197
  else:
198
  progress(0.2, desc="Generating audio with generic voice...")
199
+ audio_path = generate_voice_edgetts(script)
200
  voice_note = "Used generic voice (no voice sample provided)"
201
 
202
  progress(0.5, desc="Running Wav2Lip lip-sync (1-5 min on CPU, ~1 min on GPU)...")
 
235
  - The face should be **clearly visible** throughout the video
236
  """
237
 
238
+ STATUS_MSG = (
239
+ "Models ready. ZeroGPU enabled for fast processing."
240
+ if SETUP_OK
241
+ else f"Setup error: {SETUP_ERR}"
242
+ )
243
 
244
  css = """
245
  body { font-family: 'Inter', sans-serif; }
 
262
  with gr.Row():
263
  with gr.Column(scale=3):
264
  gr.Markdown("### Step 1 β€” Upload Your Face Video")
265
+ video_input = gr.Video(
266
  label="Face Video (MP4 / MKV)",
267
+ sources=["upload"],
268
  )
269
 
270
  gr.Markdown("### Step 2 β€” Upload Your Voice Sample")
 
278
  value=True,
279
  )
280
 
281
+ gr.Markdown("### Step 3 β€” Type Your Script")
282
  script_input = gr.Textbox(
283
  label="Script",
284
  placeholder="Hey [Name], I've been watching what you're doing and honestly there's a way to 10x your outreach using AI. Worth 2 minutes of your time?",
285
  lines=6,
286
  max_lines=20,
287
  )
 
 
 
 
 
 
288
 
289
  generate_btn = gr.Button(
290
  "Generate Lip-Synced Video",
 
294
 
295
  with gr.Column(scale=2):
296
  gr.Markdown("### Output")
297
+ video_output = gr.Video(label="Your Lip-Synced Video")
298
  status_text = gr.Textbox(label="Status", interactive=False)
299
 
300
  gr.Markdown(INSTRUCTIONS)
 
305
  )
306
 
307
  generate_btn.click(
308
+ fn=lipkit_generate,
309
+ inputs=[video_input, voice_input, script_input, use_clone],
310
  outputs=[video_output, status_text],
 
311
  )
312
 
313
  gr.Examples(
314
  examples=[
315
+ [None, None, "Hey, I've been keeping an eye on your content and honestly β€” the engagement you're getting is impressive. I built an AI tool that could help you turn one video into 50 personalized ones automatically. Want to see how it works?"],
316
+ [None, None, "What's up! I help fintech founders like you turn one video into a hundred personalized outreach clips using AI. No editing skills needed, no extra time. Want me to show you a quick demo?"],
317
  ],
318
+ inputs=[video_input, voice_input, script_input],
319
  label="Script Examples",
320
  )
321