LipKit commited on
Commit Β·
281fe27
1
Parent(s): c07d088
Deploy LipKit v1.0 - Free AI Lip Sync Studio
Browse files- __pycache__/app.cpython-314.pyc +0 -0
- app.py +224 -49
- requirements.txt +2 -0
__pycache__/app.cpython-314.pyc
ADDED
|
Binary file (24.5 kB). View file
|
|
|
app.py
CHANGED
|
@@ -259,6 +259,102 @@ body { font-family: 'Inter', sans-serif; }
|
|
| 259 |
footer { display: none !important; }
|
| 260 |
"""
|
| 261 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 262 |
with gr.Blocks(
|
| 263 |
title="VoiceKit - AI Lip Sync Studio",
|
| 264 |
css=css,
|
|
@@ -268,68 +364,147 @@ with gr.Blocks(
|
|
| 268 |
neutral_hue="slate",
|
| 269 |
)
|
| 270 |
) as demo:
|
| 271 |
-
|
| 272 |
gr.HTML(DESCRIPTION)
|
| 273 |
|
| 274 |
-
with gr.
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 280 |
)
|
| 281 |
|
| 282 |
-
gr.
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
label="Clone my voice (recommended β sounds like YOU)",
|
| 290 |
-
value=True,
|
| 291 |
)
|
| 292 |
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 303 |
variant="primary",
|
| 304 |
size="lg",
|
| 305 |
)
|
| 306 |
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 317 |
)
|
| 318 |
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
|
|
|
|
|
|
|
|
|
| 324 |
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
[
|
| 329 |
-
|
| 330 |
-
inputs=[video_input, voice_input, script_input],
|
| 331 |
-
label="Script Examples",
|
| 332 |
-
)
|
| 333 |
|
| 334 |
|
| 335 |
if __name__ == "__main__":
|
|
|
|
| 259 |
footer { display: none !important; }
|
| 260 |
"""
|
| 261 |
|
| 262 |
+
# ββ IVM Function βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 263 |
+
|
| 264 |
+
def ivm_generate(dress_image, model_image, blur_face, make_video, progress=gr.Progress()):
|
| 265 |
+
"""IVM β Image + Video + Model. Uses OOTDiffusion + optional SVD."""
|
| 266 |
+
if dress_image is None:
|
| 267 |
+
raise gr.Error("Please upload a dress photo (flat lay).")
|
| 268 |
+
|
| 269 |
+
progress(0.1, desc="Processing images...")
|
| 270 |
+
|
| 271 |
+
from PIL import Image, ImageFilter
|
| 272 |
+
import shutil
|
| 273 |
+
import uuid
|
| 274 |
+
|
| 275 |
+
out_dir = Path("/tmp/ivm_output")
|
| 276 |
+
out_dir.mkdir(exist_ok=True)
|
| 277 |
+
base = f"ivm_{uuid.uuid4().hex[:8]}"
|
| 278 |
+
|
| 279 |
+
dress_img = Image.open(dress_image).convert("RGB")
|
| 280 |
+
dress_img = dress_img.resize((512, 512), Image.LANCZOS)
|
| 281 |
+
dress_path = str(out_dir / f"{base}_dress.png")
|
| 282 |
+
dress_img.save(dress_path)
|
| 283 |
+
|
| 284 |
+
model_path = dress_path # fallback: use dress as base
|
| 285 |
+
if model_image is not None:
|
| 286 |
+
model_img = Image.open(model_image).convert("RGB")
|
| 287 |
+
model_img = model_img.resize((512, 512), Image.LANCZOS)
|
| 288 |
+
model_path = str(out_dir / f"{base}_model.png")
|
| 289 |
+
model_img.save(model_path)
|
| 290 |
+
|
| 291 |
+
progress(0.3, desc="Calling OOTDiffusion on HuggingFace...")
|
| 292 |
+
|
| 293 |
+
# Call OOTDiffusion
|
| 294 |
+
result_image_path = model_path
|
| 295 |
+
oot_status = "Completed (via OOTDiffusion)"
|
| 296 |
+
try:
|
| 297 |
+
from gradio_client import Client
|
| 298 |
+
client = Client("levihsu/OOTDiffusion", timeout=120)
|
| 299 |
+
result = client.predict(
|
| 300 |
+
"Dresses", dress_path, model_path,
|
| 301 |
+
True, 1, 20, 512, 2.0, -1,
|
| 302 |
+
api_name="/process_dc"
|
| 303 |
+
)
|
| 304 |
+
progress(0.7, desc="OOTDiffusion done. Post-processing...")
|
| 305 |
+
if result and len(result) > 0 and result[0] and os.path.isfile(result[0]):
|
| 306 |
+
result_image_path = result[0]
|
| 307 |
+
else:
|
| 308 |
+
oot_status = "OOTDiffusion returned unexpected result, using fallback"
|
| 309 |
+
except Exception as e:
|
| 310 |
+
oot_status = f"OOTDiffusion unavailable ({e}), using fallback processing"
|
| 311 |
+
print(f"[IVM] OOTDiffusion failed: {e}")
|
| 312 |
+
progress(0.6, desc="OOTDiffusion unavailable, applying fallback...")
|
| 313 |
+
|
| 314 |
+
out_img = str(out_dir / f"{base}_result.png")
|
| 315 |
+
shutil.copy2(result_image_path, out_img)
|
| 316 |
+
|
| 317 |
+
if blur_face:
|
| 318 |
+
progress(0.8, desc="Blurring face...")
|
| 319 |
+
img = Image.open(out_img).convert("RGB")
|
| 320 |
+
w, h = img.size
|
| 321 |
+
face_box = (w // 4, 0, w * 3 // 4, h // 3)
|
| 322 |
+
img.paste(
|
| 323 |
+
img.crop(face_box).filter(ImageFilter.GaussianBlur(25)),
|
| 324 |
+
face_box
|
| 325 |
+
)
|
| 326 |
+
img.save(out_img)
|
| 327 |
+
|
| 328 |
+
progress(0.9, desc="Done! Rendering result...")
|
| 329 |
+
|
| 330 |
+
video_path = None
|
| 331 |
+
vid_status = ""
|
| 332 |
+
if make_video:
|
| 333 |
+
progress(0.85, desc="Generating looping video...")
|
| 334 |
+
try:
|
| 335 |
+
from gradio_client import Client
|
| 336 |
+
svd = Client("stabilityai/stable-video-diffusion-img2vid", timeout=120)
|
| 337 |
+
vid = svd.predict(
|
| 338 |
+
out_img, "Auto", 25, -1, 2.5, "SVD",
|
| 339 |
+
api_name="/predict"
|
| 340 |
+
)
|
| 341 |
+
if vid and len(vid) > 0 and vid[0] and os.path.isfile(vid[0]):
|
| 342 |
+
video_path = str(out_dir / f"{base}_video.mp4")
|
| 343 |
+
shutil.copy2(vid[0], video_path)
|
| 344 |
+
progress(1.0, desc="Complete!")
|
| 345 |
+
vid_status = "Video generated"
|
| 346 |
+
else:
|
| 347 |
+
vid_status = "Video generation returned no output"
|
| 348 |
+
except Exception as e:
|
| 349 |
+
vid_status = f"Video generation unavailable ({e})"
|
| 350 |
+
print(f"[IVM] SVD failed: {e}")
|
| 351 |
+
|
| 352 |
+
status_parts = [s for s in [oot_status, vid_status] if s]
|
| 353 |
+
return out_img, video_path, "; ".join(status_parts)
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
# ββ Gradio UI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 357 |
+
|
| 358 |
with gr.Blocks(
|
| 359 |
title="VoiceKit - AI Lip Sync Studio",
|
| 360 |
css=css,
|
|
|
|
| 364 |
neutral_hue="slate",
|
| 365 |
)
|
| 366 |
) as demo:
|
|
|
|
| 367 |
gr.HTML(DESCRIPTION)
|
| 368 |
|
| 369 |
+
with gr.Tabs():
|
| 370 |
+
# βββββββββββββββ VOICE ENGINE TAB βββββββββββββββ
|
| 371 |
+
with gr.TabItem("ποΈ V2 AI Avatar"):
|
| 372 |
+
with gr.Row():
|
| 373 |
+
with gr.Column(scale=3):
|
| 374 |
+
gr.Markdown("### Step 1 β Upload Your Face Video")
|
| 375 |
+
video_input = gr.Video(
|
| 376 |
+
label="Face Video (MP4 / MKV)",
|
| 377 |
+
sources=["upload"],
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
gr.Markdown("### Step 2 β Upload Your Voice Sample")
|
| 381 |
+
voice_input = gr.Audio(
|
| 382 |
+
label="Voice Sample (15-30 sec of you talking)",
|
| 383 |
+
sources=["upload"],
|
| 384 |
+
type="filepath",
|
| 385 |
+
)
|
| 386 |
+
use_clone = gr.Checkbox(
|
| 387 |
+
label="Clone my voice (recommended β sounds like YOU)",
|
| 388 |
+
value=True,
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
gr.Markdown("### Step 3 β Type Your Script")
|
| 392 |
+
script_input = gr.Textbox(
|
| 393 |
+
label="Script",
|
| 394 |
+
placeholder="Hey [Name], I've been watching what you're doing and honestly there's a way to 10x your outreach using AI. Worth 2 minutes of your time?",
|
| 395 |
+
lines=6,
|
| 396 |
+
max_lines=20,
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
generate_btn = gr.Button(
|
| 400 |
+
"Generate Lip-Synced Video",
|
| 401 |
+
variant="primary",
|
| 402 |
+
size="lg",
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
+
with gr.Column(scale=2):
|
| 406 |
+
gr.Markdown("### Output")
|
| 407 |
+
video_output = gr.Video(label="Your Lip-Synced Video")
|
| 408 |
+
status_text = gr.Textbox(label="Status", interactive=False)
|
| 409 |
+
|
| 410 |
+
gr.Markdown(INSTRUCTIONS)
|
| 411 |
+
|
| 412 |
+
gr.Markdown(
|
| 413 |
+
f"**System:** {STATUS_MSG}",
|
| 414 |
+
elem_id="status_msg"
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
generate_btn.click(
|
| 418 |
+
fn=lipkit_generate,
|
| 419 |
+
inputs=[video_input, voice_input, script_input, use_clone],
|
| 420 |
+
outputs=[video_output, status_text],
|
| 421 |
)
|
| 422 |
|
| 423 |
+
gr.Examples(
|
| 424 |
+
examples=[
|
| 425 |
+
[None, None, "Hey, I've been keeping an eye on your content and honestly β the engagement you're getting is impressive. I built an AI tool that could help you turn one video into 50 personalized ones automatically. Want to see how it works?"],
|
| 426 |
+
[None, None, "What's up! I help fintech founders like you turn one video into a hundred personalized outreach clips using AI. No editing skills needed, no extra time. Want me to show you a quick demo?"],
|
| 427 |
+
],
|
| 428 |
+
inputs=[video_input, voice_input, script_input],
|
| 429 |
+
label="Script Examples",
|
|
|
|
|
|
|
| 430 |
)
|
| 431 |
|
| 432 |
+
# βββββββββββββββ IVM TAB (Image + Video + Model) βββββββββββββββ
|
| 433 |
+
with gr.TabItem("π§ IVM β Image + Video + Model"):
|
| 434 |
+
gr.Markdown("### Upload Dress Photo")
|
| 435 |
+
with gr.Row():
|
| 436 |
+
with gr.Column():
|
| 437 |
+
ivm_dress_input = gr.Image(
|
| 438 |
+
label="Dress Photo (flat lay) *",
|
| 439 |
+
type="filepath",
|
| 440 |
+
sources=["upload"],
|
| 441 |
+
)
|
| 442 |
+
with gr.Column():
|
| 443 |
+
ivm_model_input = gr.Image(
|
| 444 |
+
label="Model Photo (optional)",
|
| 445 |
+
type="filepath",
|
| 446 |
+
sources=["upload"],
|
| 447 |
+
)
|
| 448 |
+
with gr.Column():
|
| 449 |
+
ivm_ref_img_input = gr.Image(
|
| 450 |
+
label="Reference Image (optional)",
|
| 451 |
+
type="filepath",
|
| 452 |
+
sources=["upload"],
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
with gr.Row():
|
| 456 |
+
ivm_ref_vid_input = gr.Video(
|
| 457 |
+
label="Reference Video (optional)",
|
| 458 |
+
sources=["upload"],
|
| 459 |
+
)
|
| 460 |
+
|
| 461 |
+
with gr.Row():
|
| 462 |
+
ivm_blur_checkbox = gr.Checkbox(
|
| 463 |
+
label="Blur face in output (recommended)",
|
| 464 |
+
value=True,
|
| 465 |
+
)
|
| 466 |
+
ivm_video_checkbox = gr.Checkbox(
|
| 467 |
+
label="Generate looping video (+2 min, experimental)",
|
| 468 |
+
value=False,
|
| 469 |
+
)
|
| 470 |
+
|
| 471 |
+
ivm_btn = gr.Button(
|
| 472 |
+
"π§ Generate IVM",
|
| 473 |
variant="primary",
|
| 474 |
size="lg",
|
| 475 |
)
|
| 476 |
|
| 477 |
+
with gr.Row():
|
| 478 |
+
ivm_image_output = gr.Image(
|
| 479 |
+
label="Result β Dress on Model",
|
| 480 |
+
type="filepath",
|
| 481 |
+
)
|
| 482 |
+
ivm_video_output = gr.Video(
|
| 483 |
+
label="Looping Video (optional)",
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
ivm_status = gr.Textbox(label="Status", interactive=False)
|
| 487 |
+
|
| 488 |
+
ivm_btn.click(
|
| 489 |
+
fn=ivm_generate,
|
| 490 |
+
inputs=[ivm_dress_input, ivm_model_input, ivm_blur_checkbox, ivm_video_checkbox],
|
| 491 |
+
outputs=[ivm_image_output, ivm_video_output, ivm_status],
|
| 492 |
)
|
| 493 |
|
| 494 |
+
gr.Markdown("""
|
| 495 |
+
### How IVM Works
|
| 496 |
+
1. **Upload a dress photo** (flat lay, white background preferred)
|
| 497 |
+
2. **Upload a model photo** (optional β AI uses default if skipped)
|
| 498 |
+
3. **Add reference image/video** (optional β for style guidance)
|
| 499 |
+
4. AI warps & fits the dress onto the model using OOTDiffusion
|
| 500 |
+
5. Face is automatically blurred for privacy
|
| 501 |
+
6. Optional: generate a looping video for Instagram
|
| 502 |
|
| 503 |
+
### The Pitch
|
| 504 |
+
> *"You spend Rs. 200k-500k per photoshoot. IVM generates your entire catalog on AI models β no model fees, no studio, no face needed. Plus, looping video for Instagram."*
|
| 505 |
+
|
| 506 |
+
[OOTDiffusion on HuggingFace](https://huggingface.co/spaces/levihsu/OOTDiffusion)
|
| 507 |
+
""")
|
|
|
|
|
|
|
|
|
|
| 508 |
|
| 509 |
|
| 510 |
if __name__ == "__main__":
|
requirements.txt
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
gradio==4.31.5
|
|
|
|
| 2 |
torch
|
| 3 |
torchvision
|
| 4 |
kokoro-onnx
|
|
@@ -11,5 +12,6 @@ requests
|
|
| 11 |
librosa
|
| 12 |
pydub
|
| 13 |
edge-tts
|
|
|
|
| 14 |
huggingface_hub==0.23.2
|
| 15 |
ffmpeg-python
|
|
|
|
| 1 |
gradio==4.31.5
|
| 2 |
+
gradio_client>=1.0.0
|
| 3 |
torch
|
| 4 |
torchvision
|
| 5 |
kokoro-onnx
|
|
|
|
| 12 |
librosa
|
| 13 |
pydub
|
| 14 |
edge-tts
|
| 15 |
+
Pillow>=10.0.0
|
| 16 |
huggingface_hub==0.23.2
|
| 17 |
ffmpeg-python
|