LipKit commited on
Commit
281fe27
Β·
1 Parent(s): c07d088

Deploy LipKit v1.0 - Free AI Lip Sync Studio

Browse files
Files changed (3) hide show
  1. __pycache__/app.cpython-314.pyc +0 -0
  2. app.py +224 -49
  3. requirements.txt +2 -0
__pycache__/app.cpython-314.pyc ADDED
Binary file (24.5 kB). View file
 
app.py CHANGED
@@ -259,6 +259,102 @@ body { font-family: 'Inter', sans-serif; }
259
  footer { display: none !important; }
260
  """
261
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
262
  with gr.Blocks(
263
  title="VoiceKit - AI Lip Sync Studio",
264
  css=css,
@@ -268,68 +364,147 @@ with gr.Blocks(
268
  neutral_hue="slate",
269
  )
270
  ) as demo:
271
-
272
  gr.HTML(DESCRIPTION)
273
 
274
- with gr.Row():
275
- with gr.Column(scale=3):
276
- gr.Markdown("### Step 1 β€” Upload Your Face Video")
277
- video_input = gr.Video(
278
- label="Face Video (MP4 / MKV)",
279
- sources=["upload"],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
280
  )
281
 
282
- gr.Markdown("### Step 2 β€” Upload Your Voice Sample")
283
- voice_input = gr.Audio(
284
- label="Voice Sample (15-30 sec of you talking)",
285
- sources=["upload"],
286
- type="filepath",
287
- )
288
- use_clone = gr.Checkbox(
289
- label="Clone my voice (recommended β€” sounds like YOU)",
290
- value=True,
291
  )
292
 
293
- gr.Markdown("### Step 3 β€” Type Your Script")
294
- script_input = gr.Textbox(
295
- label="Script",
296
- placeholder="Hey [Name], I've been watching what you're doing and honestly there's a way to 10x your outreach using AI. Worth 2 minutes of your time?",
297
- lines=6,
298
- max_lines=20,
299
- )
300
-
301
- generate_btn = gr.Button(
302
- "Generate Lip-Synced Video",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  variant="primary",
304
  size="lg",
305
  )
306
 
307
- with gr.Column(scale=2):
308
- gr.Markdown("### Output")
309
- video_output = gr.Video(label="Your Lip-Synced Video")
310
- status_text = gr.Textbox(label="Status", interactive=False)
311
-
312
- gr.Markdown(INSTRUCTIONS)
313
-
314
- gr.Markdown(
315
- f"**System:** {STATUS_MSG}",
316
- elem_id="status_msg"
 
 
 
 
 
317
  )
318
 
319
- generate_btn.click(
320
- fn=lipkit_generate,
321
- inputs=[video_input, voice_input, script_input, use_clone],
322
- outputs=[video_output, status_text],
323
- )
 
 
 
324
 
325
- gr.Examples(
326
- examples=[
327
- [None, None, "Hey, I've been keeping an eye on your content and honestly β€” the engagement you're getting is impressive. I built an AI tool that could help you turn one video into 50 personalized ones automatically. Want to see how it works?"],
328
- [None, None, "What's up! I help fintech founders like you turn one video into a hundred personalized outreach clips using AI. No editing skills needed, no extra time. Want me to show you a quick demo?"],
329
- ],
330
- inputs=[video_input, voice_input, script_input],
331
- label="Script Examples",
332
- )
333
 
334
 
335
  if __name__ == "__main__":
 
259
  footer { display: none !important; }
260
  """
261
 
262
+ # ── IVM Function ───────────────────────────────────────────────────────────────
263
+
264
+ def ivm_generate(dress_image, model_image, blur_face, make_video, progress=gr.Progress()):
265
+ """IVM β€” Image + Video + Model. Uses OOTDiffusion + optional SVD."""
266
+ if dress_image is None:
267
+ raise gr.Error("Please upload a dress photo (flat lay).")
268
+
269
+ progress(0.1, desc="Processing images...")
270
+
271
+ from PIL import Image, ImageFilter
272
+ import shutil
273
+ import uuid
274
+
275
+ out_dir = Path("/tmp/ivm_output")
276
+ out_dir.mkdir(exist_ok=True)
277
+ base = f"ivm_{uuid.uuid4().hex[:8]}"
278
+
279
+ dress_img = Image.open(dress_image).convert("RGB")
280
+ dress_img = dress_img.resize((512, 512), Image.LANCZOS)
281
+ dress_path = str(out_dir / f"{base}_dress.png")
282
+ dress_img.save(dress_path)
283
+
284
+ model_path = dress_path # fallback: use dress as base
285
+ if model_image is not None:
286
+ model_img = Image.open(model_image).convert("RGB")
287
+ model_img = model_img.resize((512, 512), Image.LANCZOS)
288
+ model_path = str(out_dir / f"{base}_model.png")
289
+ model_img.save(model_path)
290
+
291
+ progress(0.3, desc="Calling OOTDiffusion on HuggingFace...")
292
+
293
+ # Call OOTDiffusion
294
+ result_image_path = model_path
295
+ oot_status = "Completed (via OOTDiffusion)"
296
+ try:
297
+ from gradio_client import Client
298
+ client = Client("levihsu/OOTDiffusion", timeout=120)
299
+ result = client.predict(
300
+ "Dresses", dress_path, model_path,
301
+ True, 1, 20, 512, 2.0, -1,
302
+ api_name="/process_dc"
303
+ )
304
+ progress(0.7, desc="OOTDiffusion done. Post-processing...")
305
+ if result and len(result) > 0 and result[0] and os.path.isfile(result[0]):
306
+ result_image_path = result[0]
307
+ else:
308
+ oot_status = "OOTDiffusion returned unexpected result, using fallback"
309
+ except Exception as e:
310
+ oot_status = f"OOTDiffusion unavailable ({e}), using fallback processing"
311
+ print(f"[IVM] OOTDiffusion failed: {e}")
312
+ progress(0.6, desc="OOTDiffusion unavailable, applying fallback...")
313
+
314
+ out_img = str(out_dir / f"{base}_result.png")
315
+ shutil.copy2(result_image_path, out_img)
316
+
317
+ if blur_face:
318
+ progress(0.8, desc="Blurring face...")
319
+ img = Image.open(out_img).convert("RGB")
320
+ w, h = img.size
321
+ face_box = (w // 4, 0, w * 3 // 4, h // 3)
322
+ img.paste(
323
+ img.crop(face_box).filter(ImageFilter.GaussianBlur(25)),
324
+ face_box
325
+ )
326
+ img.save(out_img)
327
+
328
+ progress(0.9, desc="Done! Rendering result...")
329
+
330
+ video_path = None
331
+ vid_status = ""
332
+ if make_video:
333
+ progress(0.85, desc="Generating looping video...")
334
+ try:
335
+ from gradio_client import Client
336
+ svd = Client("stabilityai/stable-video-diffusion-img2vid", timeout=120)
337
+ vid = svd.predict(
338
+ out_img, "Auto", 25, -1, 2.5, "SVD",
339
+ api_name="/predict"
340
+ )
341
+ if vid and len(vid) > 0 and vid[0] and os.path.isfile(vid[0]):
342
+ video_path = str(out_dir / f"{base}_video.mp4")
343
+ shutil.copy2(vid[0], video_path)
344
+ progress(1.0, desc="Complete!")
345
+ vid_status = "Video generated"
346
+ else:
347
+ vid_status = "Video generation returned no output"
348
+ except Exception as e:
349
+ vid_status = f"Video generation unavailable ({e})"
350
+ print(f"[IVM] SVD failed: {e}")
351
+
352
+ status_parts = [s for s in [oot_status, vid_status] if s]
353
+ return out_img, video_path, "; ".join(status_parts)
354
+
355
+
356
+ # ── Gradio UI ──────────────────────────────────────────────────────────────────
357
+
358
  with gr.Blocks(
359
  title="VoiceKit - AI Lip Sync Studio",
360
  css=css,
 
364
  neutral_hue="slate",
365
  )
366
  ) as demo:
 
367
  gr.HTML(DESCRIPTION)
368
 
369
+ with gr.Tabs():
370
+ # ═══════════════ VOICE ENGINE TAB ═══════════════
371
+ with gr.TabItem("πŸŽ™οΈ V2 AI Avatar"):
372
+ with gr.Row():
373
+ with gr.Column(scale=3):
374
+ gr.Markdown("### Step 1 β€” Upload Your Face Video")
375
+ video_input = gr.Video(
376
+ label="Face Video (MP4 / MKV)",
377
+ sources=["upload"],
378
+ )
379
+
380
+ gr.Markdown("### Step 2 β€” Upload Your Voice Sample")
381
+ voice_input = gr.Audio(
382
+ label="Voice Sample (15-30 sec of you talking)",
383
+ sources=["upload"],
384
+ type="filepath",
385
+ )
386
+ use_clone = gr.Checkbox(
387
+ label="Clone my voice (recommended β€” sounds like YOU)",
388
+ value=True,
389
+ )
390
+
391
+ gr.Markdown("### Step 3 β€” Type Your Script")
392
+ script_input = gr.Textbox(
393
+ label="Script",
394
+ placeholder="Hey [Name], I've been watching what you're doing and honestly there's a way to 10x your outreach using AI. Worth 2 minutes of your time?",
395
+ lines=6,
396
+ max_lines=20,
397
+ )
398
+
399
+ generate_btn = gr.Button(
400
+ "Generate Lip-Synced Video",
401
+ variant="primary",
402
+ size="lg",
403
+ )
404
+
405
+ with gr.Column(scale=2):
406
+ gr.Markdown("### Output")
407
+ video_output = gr.Video(label="Your Lip-Synced Video")
408
+ status_text = gr.Textbox(label="Status", interactive=False)
409
+
410
+ gr.Markdown(INSTRUCTIONS)
411
+
412
+ gr.Markdown(
413
+ f"**System:** {STATUS_MSG}",
414
+ elem_id="status_msg"
415
+ )
416
+
417
+ generate_btn.click(
418
+ fn=lipkit_generate,
419
+ inputs=[video_input, voice_input, script_input, use_clone],
420
+ outputs=[video_output, status_text],
421
  )
422
 
423
+ gr.Examples(
424
+ examples=[
425
+ [None, None, "Hey, I've been keeping an eye on your content and honestly β€” the engagement you're getting is impressive. I built an AI tool that could help you turn one video into 50 personalized ones automatically. Want to see how it works?"],
426
+ [None, None, "What's up! I help fintech founders like you turn one video into a hundred personalized outreach clips using AI. No editing skills needed, no extra time. Want me to show you a quick demo?"],
427
+ ],
428
+ inputs=[video_input, voice_input, script_input],
429
+ label="Script Examples",
 
 
430
  )
431
 
432
+ # ═══════════════ IVM TAB (Image + Video + Model) ═══════════════
433
+ with gr.TabItem("🧠 IVM β€” Image + Video + Model"):
434
+ gr.Markdown("### Upload Dress Photo")
435
+ with gr.Row():
436
+ with gr.Column():
437
+ ivm_dress_input = gr.Image(
438
+ label="Dress Photo (flat lay) *",
439
+ type="filepath",
440
+ sources=["upload"],
441
+ )
442
+ with gr.Column():
443
+ ivm_model_input = gr.Image(
444
+ label="Model Photo (optional)",
445
+ type="filepath",
446
+ sources=["upload"],
447
+ )
448
+ with gr.Column():
449
+ ivm_ref_img_input = gr.Image(
450
+ label="Reference Image (optional)",
451
+ type="filepath",
452
+ sources=["upload"],
453
+ )
454
+
455
+ with gr.Row():
456
+ ivm_ref_vid_input = gr.Video(
457
+ label="Reference Video (optional)",
458
+ sources=["upload"],
459
+ )
460
+
461
+ with gr.Row():
462
+ ivm_blur_checkbox = gr.Checkbox(
463
+ label="Blur face in output (recommended)",
464
+ value=True,
465
+ )
466
+ ivm_video_checkbox = gr.Checkbox(
467
+ label="Generate looping video (+2 min, experimental)",
468
+ value=False,
469
+ )
470
+
471
+ ivm_btn = gr.Button(
472
+ "🧠 Generate IVM",
473
  variant="primary",
474
  size="lg",
475
  )
476
 
477
+ with gr.Row():
478
+ ivm_image_output = gr.Image(
479
+ label="Result β€” Dress on Model",
480
+ type="filepath",
481
+ )
482
+ ivm_video_output = gr.Video(
483
+ label="Looping Video (optional)",
484
+ )
485
+
486
+ ivm_status = gr.Textbox(label="Status", interactive=False)
487
+
488
+ ivm_btn.click(
489
+ fn=ivm_generate,
490
+ inputs=[ivm_dress_input, ivm_model_input, ivm_blur_checkbox, ivm_video_checkbox],
491
+ outputs=[ivm_image_output, ivm_video_output, ivm_status],
492
  )
493
 
494
+ gr.Markdown("""
495
+ ### How IVM Works
496
+ 1. **Upload a dress photo** (flat lay, white background preferred)
497
+ 2. **Upload a model photo** (optional β€” AI uses default if skipped)
498
+ 3. **Add reference image/video** (optional β€” for style guidance)
499
+ 4. AI warps & fits the dress onto the model using OOTDiffusion
500
+ 5. Face is automatically blurred for privacy
501
+ 6. Optional: generate a looping video for Instagram
502
 
503
+ ### The Pitch
504
+ > *"You spend Rs. 200k-500k per photoshoot. IVM generates your entire catalog on AI models β€” no model fees, no studio, no face needed. Plus, looping video for Instagram."*
505
+
506
+ [OOTDiffusion on HuggingFace](https://huggingface.co/spaces/levihsu/OOTDiffusion)
507
+ """)
 
 
 
508
 
509
 
510
  if __name__ == "__main__":
requirements.txt CHANGED
@@ -1,4 +1,5 @@
1
  gradio==4.31.5
 
2
  torch
3
  torchvision
4
  kokoro-onnx
@@ -11,5 +12,6 @@ requests
11
  librosa
12
  pydub
13
  edge-tts
 
14
  huggingface_hub==0.23.2
15
  ffmpeg-python
 
1
  gradio==4.31.5
2
+ gradio_client>=1.0.0
3
  torch
4
  torchvision
5
  kokoro-onnx
 
12
  librosa
13
  pydub
14
  edge-tts
15
+ Pillow>=10.0.0
16
  huggingface_hub==0.23.2
17
  ffmpeg-python