LipKit commited on
Commit
430e263
Β·
1 Parent(s): 374f6ea

Deploy LipKit v1.0 - Free AI Lip Sync Studio

Browse files
Files changed (2) hide show
  1. __pycache__/app.cpython-314.pyc +0 -0
  2. app.py +72 -110
__pycache__/app.cpython-314.pyc CHANGED
Binary files a/__pycache__/app.cpython-314.pyc and b/__pycache__/app.cpython-314.pyc differ
 
app.py CHANGED
@@ -197,127 +197,89 @@ def lipkit_generate(video_file, voice_file, script, use_voice_clone, language, p
197
  return output_path, f"Done! {voice_note}"
198
 
199
  # ── IVM Functions ─────────────────────────────────────────────────────────────
200
- CACHED_OOTD = Path("/tmp/ootd_checkpoints/ootd.pt")
201
 
202
  def ivm_generate_image(dress_image, model_image, progress=gr.Progress()):
203
- """IVM β€” Generate dress-on-model. Tries API backends, falls back to local CPU."""
204
  if dress_image is None: raise gr.Error("Please upload a dress photo.")
205
  progress(0.1, desc="Processing images...")
206
  from PIL import Image
207
- import uuid, requests as req_lib
208
  base = f"ivm_{uuid.uuid4().hex[:8]}"
209
 
210
- dress_img = Image.open(dress_image).convert("RGB")
211
- dress_img = dress_img.resize((768, 1024), Image.LANCZOS)
212
- dress_path = str(IVM_DIR / f"{base}_dress.png"); dress_img.save(dress_path)
213
 
214
- model_path = dress_path
215
- if model_image is not None:
216
- m = Image.open(model_image).convert("RGB").resize((768, 1024), Image.LANCZOS)
217
- model_path = str(IVM_DIR / f"{base}_model.png"); m.save(model_path)
218
 
219
- result = None
220
- errors = []
221
 
222
- # 1) Try Modal backend first (our own GPU)
223
- modal_url = os.environ.get("MODAL_VTON_ENDPOINT", "").strip()
224
- if modal_url:
225
- try:
226
- import cloudinary, cloudinary.uploader
227
- cloudinary.config(cloud_name=os.environ.get("CLOUDINARY_CLOUD_NAME",""),
228
- api_key=os.environ.get("CLOUDINARY_API_KEY",""),
229
- api_secret=os.environ.get("CLOUDINARY_API_SECRET",""))
230
- du = cloudinary.uploader.upload(dress_path, resource_type="image")
231
- mu = cloudinary.uploader.upload(model_path, resource_type="image")
232
- resp = req_lib.post(modal_url, json={
233
- "dress_image_url": du["secure_url"], "model_image_url": mu["secure_url"],
234
- "blur_face": False,
235
- }, timeout=600)
236
- if resp.status_code == 200 and resp.json().get("result_url"):
237
- r2 = req_lib.get(resp.json()["result_url"], timeout=120)
238
- result = str(IVM_DIR / f"{base}_modal.png")
239
- with open(result, "wb") as f: f.write(r2.content)
240
- except Exception as e: errors.append(f"Modal: {e}")
241
-
242
- # 2) Try Gradio backends
243
- if not result:
244
- for name, url, api, fmt in [
245
- ("tonyassi/fashion-try-on", "tonyassi/fashion-try-on", "/generate", "dict"),
246
- ("wizzseen/virtual-try-on", "wizzseen/virtual-try-on", "/predict", "path"),
247
- ]:
248
- try:
249
- from gradio_client import Client
250
- progress(0.3, desc=f"Trying {name}...")
251
- if fmt == "dict":
252
- r = Client(url).predict({"path": model_path, "meta": {}}, {"path": dress_path, "meta": {}}, api_name=api)
253
- if r and isinstance(r, dict) and r.get("path") and os.path.isfile(r["path"]):
254
- result = str(IVM_DIR / f"{base}_gradio.png"); shutil.copy2(r["path"], result); break
255
- else:
256
- r = Client(url).predict(dress_path, model_path, api_name=api)
257
- if r and (isinstance(r, str) and os.path.isfile(r)) or (isinstance(r, dict) and r.get("path")):
258
- p = r if isinstance(r, str) else r["path"]
259
- result = str(IVM_DIR / f"{base}_gradio.png"); shutil.copy2(p, result); break
260
- except Exception as e: errors.append(f"{name}: {e}")
261
-
262
- # 3) Fallback: Run OOTDiffusion locally on CPU (slow but free)
263
- if not result:
264
- progress(0.3, desc="API backends unavailable. Running local OOTDiffusion on CPU (5-10 min)...")
265
- try:
266
- result = run_ootd_local(dress_path, model_path, base)
267
- except Exception as e:
268
- errors.append(f"Local OOT: {e}")
269
- raise gr.Error(
270
- "IVM is temporarily unavailable. All virtual try-on services are down:\n"
271
- + "\n".join(errors)
272
- )
273
-
274
- return crop_body_only(result, base)
275
-
276
-
277
- def run_ootd_local(dress_path: str, model_path: str, base: str) -> str:
278
- """Run OOTDiffusion directly on CPU. Slow (~5-10 min) but free."""
279
- OOTD_DIR = "/tmp/OOTDiffusion"
280
- ckpt_dir = "/tmp/ootd_checkpoints"
281
- os.makedirs(ckpt_dir, exist_ok=True)
282
-
283
- # Clone OOTDiffusion if not already cloned
284
- if not os.path.exists(OOTD_DIR):
285
- subprocess.run(["git", "clone", "--depth=1", "https://github.com/levihsu/OOTDiffusion.git", OOTD_DIR],
286
- check=True, capture_output=True, timeout=120)
287
- subprocess.run([sys.executable, "-m", "pip", "install", "-r", f"{OOTD_DIR}/requirements.txt", "-q"],
288
- capture_output=True, timeout=300)
289
-
290
- # Download model weights if not cached
291
- if not CACHED_OOTD.exists():
292
- from huggingface_hub import hf_hub_download
293
- hf_hub_download("levihsu/OOTDiffusion", "ootd.pt", local_dir=ckpt_dir)
294
-
295
- # Run inference
296
- cmd = [sys.executable, f"{OOTD_DIR}/run.py",
297
- "--model_type", "Dresses",
298
- "--dress_path", dress_path,
299
- "--model_path", model_path,
300
- "--output_dir", str(IVM_DIR),
301
- "--checkpoint_dir", ckpt_dir,
302
- "--device", "cpu"]
303
- subprocess.run(cmd, capture_output=True, text=True, timeout=1800)
304
-
305
- # Find output
306
- for f in sorted(IVM_DIR.glob(f"{base[:10]}*"), key=os.path.getmtime, reverse=True):
307
- if f.suffix in (".png", ".jpg"):
308
- return str(f)
309
- raise RuntimeError("Local OOTDiffusion produced no output")
310
-
311
-
312
- def crop_body_only(image_path: str, base: str) -> str:
313
- """Crop top 30% off to hide face."""
314
  from PIL import Image
315
- out_img = str(IVM_DIR / f"{base}_body.png")
316
- img = Image.open(image_path).convert("RGB")
317
- w, h = img.size
318
- img.crop((0, int(h * 0.30), w, h)).save(out_img)
319
- save_to_history({"type": "ivm", "time": time.strftime("%Y-%m-%d %H:%M"), "image_path": out_img, "body_only": True})
320
- return out_img
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
321
 
322
  def ivm_generate_video(image_path, progress=gr.Progress()):
323
  """IVM β€” Generate looping video from existing IVM image."""
 
197
  return output_path, f"Done! {voice_note}"
198
 
199
  # ── IVM Functions ─────────────────────────────────────────────────────────────
 
200
 
201
  def ivm_generate_image(dress_image, model_image, progress=gr.Progress()):
202
+ """IVM β€” Composite dress onto model body using AI pose detection + seam carving."""
203
  if dress_image is None: raise gr.Error("Please upload a dress photo.")
204
  progress(0.1, desc="Processing images...")
205
  from PIL import Image
206
+ import uuid
207
  base = f"ivm_{uuid.uuid4().hex[:8]}"
208
 
209
+ dress = Image.open(dress_image).convert("RGB")
210
+ model = Image.open(model_image).convert("RGB") if model_image else None
 
211
 
212
+ progress(0.3, desc="Detecting body pose and compositing dress...")
213
+ result = composite_dress(dress, model)
 
 
214
 
215
+ out = str(IVM_DIR / f"{base}_result.png")
216
+ result.save(out)
217
 
218
+ # Crop top 30% to hide face
219
+ progress(0.7, desc="Cropping to body-only view...")
220
+ w, h = result.size
221
+ Image.open(out).crop((0, int(h * 0.30), w, h)).save(out)
222
+
223
+ save_to_history({"type": "ivm", "time": time.strftime("%Y-%m-%d %H:%M"), "image_path": out, "body_only": True})
224
+ progress(1.0, desc="Done!")
225
+ return out
226
+
227
+
228
+ def composite_dress(dress_img, model_img=None):
229
+ """Seamlessly composite dress onto model using MediaPipe pose detection."""
230
+ import cv2
231
+ import numpy as np
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  from PIL import Image
233
+
234
+ if model_img is None:
235
+ model_img = Image.new("RGB", (768, 1024), (245, 240, 235))
236
+
237
+ dress = dress_img.resize((768, 1024), Image.LANCZOS)
238
+ model = model_img.resize((768, 1024), Image.LANCZOS)
239
+ model_cv = cv2.cvtColor(np.array(model), cv2.COLOR_RGB2BGR)
240
+
241
+ try:
242
+ import mediapipe as mp
243
+ with mp.solutions.pose.Pose(static_image_mode=True, model_complexity=1) as pose:
244
+ results = pose.process(cv2.cvtColor(model_cv, cv2.COLOR_BGR2RGB))
245
+
246
+ if results.pose_landmarks:
247
+ h, w = model_cv.shape[:2]
248
+ lm = results.pose_landmarks.landmark
249
+ # Get shoulder/hip landmarks for dress positioning
250
+ ls = (int(lm[11].x * w), int(lm[11].y * h))
251
+ rs = (int(lm[12].x * w), int(lm[12].y * h))
252
+ lh = (int(lm[23].x * w), int(lm[23].y * h))
253
+ rh = (int(lm[24].x * w), int(lm[24].y * h))
254
+
255
+ cx = (ls[0] + rs[0]) // 2
256
+ top_y = min(ls[1], rs[1])
257
+ bot_y = max(lh[1], rh[1])
258
+ body_w = abs(rs[0] - ls[0]) * 2
259
+
260
+ # Resize dress to body width with 1:1.7 aspect ratio
261
+ dress_res = dress.resize((int(body_w), int(body_w * 1.7)), Image.LANCZOS)
262
+ # Remove white background from dress
263
+ dress_cv = cv2.cvtColor(np.array(dress_res), cv2.COLOR_RGB2BGR)
264
+ gray = cv2.GaussianBlur(cv2.cvtColor(dress_cv, cv2.COLOR_BGR2GRAY), (3,3), 0)
265
+ _, mask = cv2.threshold(gray, 235, 255, cv2.THRESH_BINARY_INV)
266
+ mask = cv2.dilate(mask, None, iterations=1)
267
+
268
+ # Seamless clone
269
+ center = (cx, (top_y + bot_y) // 2)
270
+ try:
271
+ res = cv2.seamlessClone(dress_cv, model_cv, mask, center, cv2.NORMAL_CLONE)
272
+ return Image.fromarray(cv2.cvtColor(res, cv2.COLOR_BGR2RGB))
273
+ except:
274
+ pass
275
+ except Exception as e:
276
+ print(f"[IVM] Pose failed: {e}")
277
+
278
+ # Simple center composite fallback
279
+ dr = dress.resize((model.width // 2, model.height // 2), Image.LANCZOS)
280
+ r = model.copy()
281
+ r.paste(dr, ((model.width - dr.width) // 2, int(model.height * 0.15)))
282
+ return r
283
 
284
  def ivm_generate_video(image_path, progress=gr.Progress()):
285
  """IVM β€” Generate looping video from existing IVM image."""