LipKit commited on
Commit Β·
430e263
1
Parent(s): 374f6ea
Deploy LipKit v1.0 - Free AI Lip Sync Studio
Browse files- __pycache__/app.cpython-314.pyc +0 -0
- app.py +72 -110
__pycache__/app.cpython-314.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-314.pyc and b/__pycache__/app.cpython-314.pyc differ
|
|
|
app.py
CHANGED
|
@@ -197,127 +197,89 @@ def lipkit_generate(video_file, voice_file, script, use_voice_clone, language, p
|
|
| 197 |
return output_path, f"Done! {voice_note}"
|
| 198 |
|
| 199 |
# ββ IVM Functions βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 200 |
-
CACHED_OOTD = Path("/tmp/ootd_checkpoints/ootd.pt")
|
| 201 |
|
| 202 |
def ivm_generate_image(dress_image, model_image, progress=gr.Progress()):
|
| 203 |
-
"""IVM β
|
| 204 |
if dress_image is None: raise gr.Error("Please upload a dress photo.")
|
| 205 |
progress(0.1, desc="Processing images...")
|
| 206 |
from PIL import Image
|
| 207 |
-
import uuid
|
| 208 |
base = f"ivm_{uuid.uuid4().hex[:8]}"
|
| 209 |
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
dress_path = str(IVM_DIR / f"{base}_dress.png"); dress_img.save(dress_path)
|
| 213 |
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
m = Image.open(model_image).convert("RGB").resize((768, 1024), Image.LANCZOS)
|
| 217 |
-
model_path = str(IVM_DIR / f"{base}_model.png"); m.save(model_path)
|
| 218 |
|
| 219 |
-
|
| 220 |
-
|
| 221 |
|
| 222 |
-
#
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
if resp.status_code == 200 and resp.json().get("result_url"):
|
| 237 |
-
r2 = req_lib.get(resp.json()["result_url"], timeout=120)
|
| 238 |
-
result = str(IVM_DIR / f"{base}_modal.png")
|
| 239 |
-
with open(result, "wb") as f: f.write(r2.content)
|
| 240 |
-
except Exception as e: errors.append(f"Modal: {e}")
|
| 241 |
-
|
| 242 |
-
# 2) Try Gradio backends
|
| 243 |
-
if not result:
|
| 244 |
-
for name, url, api, fmt in [
|
| 245 |
-
("tonyassi/fashion-try-on", "tonyassi/fashion-try-on", "/generate", "dict"),
|
| 246 |
-
("wizzseen/virtual-try-on", "wizzseen/virtual-try-on", "/predict", "path"),
|
| 247 |
-
]:
|
| 248 |
-
try:
|
| 249 |
-
from gradio_client import Client
|
| 250 |
-
progress(0.3, desc=f"Trying {name}...")
|
| 251 |
-
if fmt == "dict":
|
| 252 |
-
r = Client(url).predict({"path": model_path, "meta": {}}, {"path": dress_path, "meta": {}}, api_name=api)
|
| 253 |
-
if r and isinstance(r, dict) and r.get("path") and os.path.isfile(r["path"]):
|
| 254 |
-
result = str(IVM_DIR / f"{base}_gradio.png"); shutil.copy2(r["path"], result); break
|
| 255 |
-
else:
|
| 256 |
-
r = Client(url).predict(dress_path, model_path, api_name=api)
|
| 257 |
-
if r and (isinstance(r, str) and os.path.isfile(r)) or (isinstance(r, dict) and r.get("path")):
|
| 258 |
-
p = r if isinstance(r, str) else r["path"]
|
| 259 |
-
result = str(IVM_DIR / f"{base}_gradio.png"); shutil.copy2(p, result); break
|
| 260 |
-
except Exception as e: errors.append(f"{name}: {e}")
|
| 261 |
-
|
| 262 |
-
# 3) Fallback: Run OOTDiffusion locally on CPU (slow but free)
|
| 263 |
-
if not result:
|
| 264 |
-
progress(0.3, desc="API backends unavailable. Running local OOTDiffusion on CPU (5-10 min)...")
|
| 265 |
-
try:
|
| 266 |
-
result = run_ootd_local(dress_path, model_path, base)
|
| 267 |
-
except Exception as e:
|
| 268 |
-
errors.append(f"Local OOT: {e}")
|
| 269 |
-
raise gr.Error(
|
| 270 |
-
"IVM is temporarily unavailable. All virtual try-on services are down:\n"
|
| 271 |
-
+ "\n".join(errors)
|
| 272 |
-
)
|
| 273 |
-
|
| 274 |
-
return crop_body_only(result, base)
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
def run_ootd_local(dress_path: str, model_path: str, base: str) -> str:
|
| 278 |
-
"""Run OOTDiffusion directly on CPU. Slow (~5-10 min) but free."""
|
| 279 |
-
OOTD_DIR = "/tmp/OOTDiffusion"
|
| 280 |
-
ckpt_dir = "/tmp/ootd_checkpoints"
|
| 281 |
-
os.makedirs(ckpt_dir, exist_ok=True)
|
| 282 |
-
|
| 283 |
-
# Clone OOTDiffusion if not already cloned
|
| 284 |
-
if not os.path.exists(OOTD_DIR):
|
| 285 |
-
subprocess.run(["git", "clone", "--depth=1", "https://github.com/levihsu/OOTDiffusion.git", OOTD_DIR],
|
| 286 |
-
check=True, capture_output=True, timeout=120)
|
| 287 |
-
subprocess.run([sys.executable, "-m", "pip", "install", "-r", f"{OOTD_DIR}/requirements.txt", "-q"],
|
| 288 |
-
capture_output=True, timeout=300)
|
| 289 |
-
|
| 290 |
-
# Download model weights if not cached
|
| 291 |
-
if not CACHED_OOTD.exists():
|
| 292 |
-
from huggingface_hub import hf_hub_download
|
| 293 |
-
hf_hub_download("levihsu/OOTDiffusion", "ootd.pt", local_dir=ckpt_dir)
|
| 294 |
-
|
| 295 |
-
# Run inference
|
| 296 |
-
cmd = [sys.executable, f"{OOTD_DIR}/run.py",
|
| 297 |
-
"--model_type", "Dresses",
|
| 298 |
-
"--dress_path", dress_path,
|
| 299 |
-
"--model_path", model_path,
|
| 300 |
-
"--output_dir", str(IVM_DIR),
|
| 301 |
-
"--checkpoint_dir", ckpt_dir,
|
| 302 |
-
"--device", "cpu"]
|
| 303 |
-
subprocess.run(cmd, capture_output=True, text=True, timeout=1800)
|
| 304 |
-
|
| 305 |
-
# Find output
|
| 306 |
-
for f in sorted(IVM_DIR.glob(f"{base[:10]}*"), key=os.path.getmtime, reverse=True):
|
| 307 |
-
if f.suffix in (".png", ".jpg"):
|
| 308 |
-
return str(f)
|
| 309 |
-
raise RuntimeError("Local OOTDiffusion produced no output")
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
def crop_body_only(image_path: str, base: str) -> str:
|
| 313 |
-
"""Crop top 30% off to hide face."""
|
| 314 |
from PIL import Image
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
|
| 322 |
def ivm_generate_video(image_path, progress=gr.Progress()):
|
| 323 |
"""IVM β Generate looping video from existing IVM image."""
|
|
|
|
| 197 |
return output_path, f"Done! {voice_note}"
|
| 198 |
|
| 199 |
# ββ IVM Functions βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 200 |
|
| 201 |
def ivm_generate_image(dress_image, model_image, progress=gr.Progress()):
|
| 202 |
+
"""IVM β Composite dress onto model body using AI pose detection + seam carving."""
|
| 203 |
if dress_image is None: raise gr.Error("Please upload a dress photo.")
|
| 204 |
progress(0.1, desc="Processing images...")
|
| 205 |
from PIL import Image
|
| 206 |
+
import uuid
|
| 207 |
base = f"ivm_{uuid.uuid4().hex[:8]}"
|
| 208 |
|
| 209 |
+
dress = Image.open(dress_image).convert("RGB")
|
| 210 |
+
model = Image.open(model_image).convert("RGB") if model_image else None
|
|
|
|
| 211 |
|
| 212 |
+
progress(0.3, desc="Detecting body pose and compositing dress...")
|
| 213 |
+
result = composite_dress(dress, model)
|
|
|
|
|
|
|
| 214 |
|
| 215 |
+
out = str(IVM_DIR / f"{base}_result.png")
|
| 216 |
+
result.save(out)
|
| 217 |
|
| 218 |
+
# Crop top 30% to hide face
|
| 219 |
+
progress(0.7, desc="Cropping to body-only view...")
|
| 220 |
+
w, h = result.size
|
| 221 |
+
Image.open(out).crop((0, int(h * 0.30), w, h)).save(out)
|
| 222 |
+
|
| 223 |
+
save_to_history({"type": "ivm", "time": time.strftime("%Y-%m-%d %H:%M"), "image_path": out, "body_only": True})
|
| 224 |
+
progress(1.0, desc="Done!")
|
| 225 |
+
return out
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def composite_dress(dress_img, model_img=None):
|
| 229 |
+
"""Seamlessly composite dress onto model using MediaPipe pose detection."""
|
| 230 |
+
import cv2
|
| 231 |
+
import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
from PIL import Image
|
| 233 |
+
|
| 234 |
+
if model_img is None:
|
| 235 |
+
model_img = Image.new("RGB", (768, 1024), (245, 240, 235))
|
| 236 |
+
|
| 237 |
+
dress = dress_img.resize((768, 1024), Image.LANCZOS)
|
| 238 |
+
model = model_img.resize((768, 1024), Image.LANCZOS)
|
| 239 |
+
model_cv = cv2.cvtColor(np.array(model), cv2.COLOR_RGB2BGR)
|
| 240 |
+
|
| 241 |
+
try:
|
| 242 |
+
import mediapipe as mp
|
| 243 |
+
with mp.solutions.pose.Pose(static_image_mode=True, model_complexity=1) as pose:
|
| 244 |
+
results = pose.process(cv2.cvtColor(model_cv, cv2.COLOR_BGR2RGB))
|
| 245 |
+
|
| 246 |
+
if results.pose_landmarks:
|
| 247 |
+
h, w = model_cv.shape[:2]
|
| 248 |
+
lm = results.pose_landmarks.landmark
|
| 249 |
+
# Get shoulder/hip landmarks for dress positioning
|
| 250 |
+
ls = (int(lm[11].x * w), int(lm[11].y * h))
|
| 251 |
+
rs = (int(lm[12].x * w), int(lm[12].y * h))
|
| 252 |
+
lh = (int(lm[23].x * w), int(lm[23].y * h))
|
| 253 |
+
rh = (int(lm[24].x * w), int(lm[24].y * h))
|
| 254 |
+
|
| 255 |
+
cx = (ls[0] + rs[0]) // 2
|
| 256 |
+
top_y = min(ls[1], rs[1])
|
| 257 |
+
bot_y = max(lh[1], rh[1])
|
| 258 |
+
body_w = abs(rs[0] - ls[0]) * 2
|
| 259 |
+
|
| 260 |
+
# Resize dress to body width with 1:1.7 aspect ratio
|
| 261 |
+
dress_res = dress.resize((int(body_w), int(body_w * 1.7)), Image.LANCZOS)
|
| 262 |
+
# Remove white background from dress
|
| 263 |
+
dress_cv = cv2.cvtColor(np.array(dress_res), cv2.COLOR_RGB2BGR)
|
| 264 |
+
gray = cv2.GaussianBlur(cv2.cvtColor(dress_cv, cv2.COLOR_BGR2GRAY), (3,3), 0)
|
| 265 |
+
_, mask = cv2.threshold(gray, 235, 255, cv2.THRESH_BINARY_INV)
|
| 266 |
+
mask = cv2.dilate(mask, None, iterations=1)
|
| 267 |
+
|
| 268 |
+
# Seamless clone
|
| 269 |
+
center = (cx, (top_y + bot_y) // 2)
|
| 270 |
+
try:
|
| 271 |
+
res = cv2.seamlessClone(dress_cv, model_cv, mask, center, cv2.NORMAL_CLONE)
|
| 272 |
+
return Image.fromarray(cv2.cvtColor(res, cv2.COLOR_BGR2RGB))
|
| 273 |
+
except:
|
| 274 |
+
pass
|
| 275 |
+
except Exception as e:
|
| 276 |
+
print(f"[IVM] Pose failed: {e}")
|
| 277 |
+
|
| 278 |
+
# Simple center composite fallback
|
| 279 |
+
dr = dress.resize((model.width // 2, model.height // 2), Image.LANCZOS)
|
| 280 |
+
r = model.copy()
|
| 281 |
+
r.paste(dr, ((model.width - dr.width) // 2, int(model.height * 0.15)))
|
| 282 |
+
return r
|
| 283 |
|
| 284 |
def ivm_generate_video(image_path, progress=gr.Progress()):
|
| 285 |
"""IVM β Generate looping video from existing IVM image."""
|