Update app.py
Browse files
app.py
CHANGED
|
@@ -14,7 +14,7 @@ import onnxruntime as ort
|
|
| 14 |
|
| 15 |
|
| 16 |
# ============================================================
|
| 17 |
-
#
|
| 18 |
# ============================================================
|
| 19 |
|
| 20 |
MODEL_PATH = "wav2lip_models/wav2lip_gan.onnx"
|
|
@@ -46,22 +46,29 @@ FACE_MODEL_URL = (
|
|
| 46 |
|
| 47 |
|
| 48 |
# ============================================================
|
| 49 |
-
#
|
| 50 |
# ============================================================
|
| 51 |
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
)
|
| 56 |
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
# ============================================================
|
| 64 |
-
# AUDIO
|
| 65 |
# ============================================================
|
| 66 |
|
| 67 |
SAMPLE_RATE = 16000
|
|
@@ -80,43 +87,33 @@ FMAX = 7600
|
|
| 80 |
|
| 81 |
MEL_STEP_SIZE = 16
|
| 82 |
|
|
|
|
| 83 |
MEL_FPS = 80.0
|
| 84 |
|
| 85 |
|
| 86 |
# ============================================================
|
| 87 |
-
#
|
| 88 |
# ============================================================
|
| 89 |
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
FACE_PADDING = 0.08
|
| 93 |
-
|
| 94 |
-
FACE_SMOOTHING = 0.20
|
| 95 |
-
|
| 96 |
-
REDETECT_SECONDS = 0.30
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
# ============================================================
|
| 100 |
-
# MOUTH
|
| 101 |
-
# ============================================================
|
| 102 |
|
| 103 |
-
|
| 104 |
-
# Too high = generated mouth becomes obvious.
|
| 105 |
-
BLEND_STRENGTH = 0.62
|
| 106 |
|
| 107 |
|
| 108 |
# ============================================================
|
| 109 |
-
#
|
| 110 |
# ============================================================
|
| 111 |
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
|
|
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
-
# ============================================================
|
| 118 |
-
# PRINT
|
| 119 |
-
# ============================================================
|
| 120 |
|
| 121 |
print("=" * 70)
|
| 122 |
print("Wav2Lip High Quality Lip Sync")
|
|
@@ -130,15 +127,16 @@ print("=" * 70)
|
|
| 130 |
if not os.path.exists(MODEL_PATH):
|
| 131 |
|
| 132 |
raise FileNotFoundError(
|
| 133 |
-
"\n\
|
|
|
|
| 134 |
f"{MODEL_PATH}\n\n"
|
| 135 |
-
"
|
| 136 |
"wav2lip_models/wav2lip_gan.onnx\n"
|
| 137 |
)
|
| 138 |
|
| 139 |
|
| 140 |
# ============================================================
|
| 141 |
-
# ONNX
|
| 142 |
# ============================================================
|
| 143 |
|
| 144 |
available_providers = (
|
|
@@ -166,25 +164,22 @@ print(
|
|
| 166 |
|
| 167 |
|
| 168 |
# ============================================================
|
| 169 |
-
# LOAD MODEL
|
| 170 |
# ============================================================
|
| 171 |
|
| 172 |
print(
|
| 173 |
"Loading Wav2Lip ONNX model..."
|
| 174 |
)
|
| 175 |
|
| 176 |
-
|
| 177 |
SESSION = ort.InferenceSession(
|
| 178 |
MODEL_PATH,
|
| 179 |
providers=ONNX_PROVIDERS
|
| 180 |
)
|
| 181 |
|
| 182 |
-
|
| 183 |
print(
|
| 184 |
"Model loaded successfully."
|
| 185 |
)
|
| 186 |
|
| 187 |
-
|
| 188 |
print(
|
| 189 |
"Model inputs:"
|
| 190 |
)
|
|
@@ -198,7 +193,6 @@ for inp in SESSION.get_inputs():
|
|
| 198 |
inp.type
|
| 199 |
)
|
| 200 |
|
| 201 |
-
|
| 202 |
print(
|
| 203 |
"Model outputs:"
|
| 204 |
)
|
|
@@ -214,7 +208,7 @@ for out in SESSION.get_outputs():
|
|
| 214 |
|
| 215 |
|
| 216 |
# ============================================================
|
| 217 |
-
# DOWNLOAD FACE
|
| 218 |
# ============================================================
|
| 219 |
|
| 220 |
def download_file(url, path):
|
|
@@ -253,49 +247,51 @@ print(
|
|
| 253 |
"Loading OpenCV DNN face detector..."
|
| 254 |
)
|
| 255 |
|
| 256 |
-
|
| 257 |
FACE_DETECTOR = cv2.dnn.readNetFromCaffe(
|
| 258 |
FACE_PROTO,
|
| 259 |
FACE_MODEL
|
| 260 |
)
|
| 261 |
|
| 262 |
-
|
| 263 |
print(
|
| 264 |
"Face detector loaded."
|
| 265 |
)
|
| 266 |
|
| 267 |
|
| 268 |
# ============================================================
|
| 269 |
-
# TEMPORAL
|
| 270 |
# ============================================================
|
| 271 |
|
| 272 |
class MouthTemporalSmoother:
|
| 273 |
|
| 274 |
def __init__(
|
| 275 |
self,
|
| 276 |
-
alpha=0.
|
| 277 |
):
|
| 278 |
|
| 279 |
self.alpha = alpha
|
| 280 |
|
| 281 |
self.previous = None
|
| 282 |
|
|
|
|
| 283 |
def reset(self):
|
| 284 |
|
| 285 |
self.previous = None
|
| 286 |
|
|
|
|
| 287 |
def smooth(
|
| 288 |
self,
|
| 289 |
generated,
|
| 290 |
mask
|
| 291 |
):
|
| 292 |
|
| 293 |
-
current = (
|
| 294 |
-
|
| 295 |
-
np.float32
|
| 296 |
-
)
|
| 297 |
)
|
| 298 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
if self.previous is None:
|
| 300 |
|
| 301 |
self.previous = current.copy()
|
|
@@ -303,34 +299,89 @@ class MouthTemporalSmoother:
|
|
| 303 |
return generated
|
| 304 |
|
| 305 |
|
| 306 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
|
| 308 |
|
| 309 |
# ----------------------------------------------------
|
| 310 |
-
# Temporal smoothing
|
| 311 |
#
|
| 312 |
-
#
|
| 313 |
-
# This keeps lip motion responsive to audio.
|
| 314 |
# ----------------------------------------------------
|
| 315 |
|
| 316 |
smoothed = (
|
| 317 |
-
|
|
|
|
|
|
|
| 318 |
+
|
| 319 |
-
current
|
| 320 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
)
|
| 322 |
)
|
| 323 |
|
| 324 |
|
| 325 |
# ----------------------------------------------------
|
| 326 |
-
# Only stabilize mouth
|
| 327 |
# ----------------------------------------------------
|
| 328 |
|
| 329 |
-
m = mask[
|
| 330 |
-
:,
|
| 331 |
-
:,
|
| 332 |
-
None
|
| 333 |
-
]
|
| 334 |
|
| 335 |
|
| 336 |
result = (
|
|
@@ -342,10 +393,7 @@ class MouthTemporalSmoother:
|
|
| 342 |
)
|
| 343 |
|
| 344 |
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
return np.clip(
|
| 349 |
result,
|
| 350 |
0,
|
| 351 |
255
|
|
@@ -354,13 +402,23 @@ class MouthTemporalSmoother:
|
|
| 354 |
)
|
| 355 |
|
| 356 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 357 |
MOUTH_SMOOTHER = MouthTemporalSmoother(
|
| 358 |
-
alpha=
|
| 359 |
)
|
| 360 |
|
| 361 |
|
| 362 |
# ============================================================
|
| 363 |
-
# AUDIO
|
| 364 |
# ============================================================
|
| 365 |
|
| 366 |
def load_audio(
|
|
@@ -379,31 +437,30 @@ def load_audio(
|
|
| 379 |
"Could not load audio."
|
| 380 |
)
|
| 381 |
|
| 382 |
-
|
| 383 |
if len(audio) == 0:
|
| 384 |
|
| 385 |
raise RuntimeError(
|
| 386 |
"Audio file is empty."
|
| 387 |
)
|
| 388 |
|
| 389 |
-
|
| 390 |
audio = audio.astype(
|
| 391 |
np.float32
|
| 392 |
)
|
| 393 |
|
| 394 |
-
|
| 395 |
duration = (
|
| 396 |
len(audio)
|
| 397 |
/
|
| 398 |
SAMPLE_RATE
|
| 399 |
)
|
| 400 |
|
| 401 |
-
|
| 402 |
-
|
|
|
|
|
|
|
| 403 |
|
| 404 |
|
| 405 |
# ============================================================
|
| 406 |
-
#
|
| 407 |
# ============================================================
|
| 408 |
|
| 409 |
def make_mel(
|
|
@@ -424,18 +481,15 @@ def make_mel(
|
|
| 424 |
center=True
|
| 425 |
)
|
| 426 |
|
| 427 |
-
|
| 428 |
mel = np.maximum(
|
| 429 |
mel,
|
| 430 |
1e-5
|
| 431 |
)
|
| 432 |
|
| 433 |
-
|
| 434 |
mel = np.log10(
|
| 435 |
mel
|
| 436 |
)
|
| 437 |
|
| 438 |
-
|
| 439 |
return mel.astype(
|
| 440 |
np.float32
|
| 441 |
)
|
|
@@ -449,7 +503,9 @@ def detect_face(
|
|
| 449 |
frame
|
| 450 |
):
|
| 451 |
|
| 452 |
-
|
|
|
|
|
|
|
| 453 |
|
| 454 |
|
| 455 |
blob = cv2.dnn.blobFromImage(
|
|
@@ -494,7 +550,7 @@ def detect_face(
|
|
| 494 |
)
|
| 495 |
|
| 496 |
|
| 497 |
-
if confidence <
|
| 498 |
|
| 499 |
continue
|
| 500 |
|
|
@@ -509,10 +565,10 @@ def detect_face(
|
|
| 509 |
*
|
| 510 |
np.array(
|
| 511 |
[
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
]
|
| 517 |
)
|
| 518 |
)
|
|
@@ -529,7 +585,7 @@ def detect_face(
|
|
| 529 |
0,
|
| 530 |
min(
|
| 531 |
x1,
|
| 532 |
-
|
| 533 |
)
|
| 534 |
)
|
| 535 |
|
|
@@ -537,7 +593,7 @@ def detect_face(
|
|
| 537 |
0,
|
| 538 |
min(
|
| 539 |
y1,
|
| 540 |
-
|
| 541 |
)
|
| 542 |
)
|
| 543 |
|
|
@@ -545,7 +601,7 @@ def detect_face(
|
|
| 545 |
0,
|
| 546 |
min(
|
| 547 |
x2,
|
| 548 |
-
|
| 549 |
)
|
| 550 |
)
|
| 551 |
|
|
@@ -553,7 +609,7 @@ def detect_face(
|
|
| 553 |
0,
|
| 554 |
min(
|
| 555 |
y2,
|
| 556 |
-
|
| 557 |
)
|
| 558 |
)
|
| 559 |
|
|
@@ -568,7 +624,11 @@ def detect_face(
|
|
| 568 |
continue
|
| 569 |
|
| 570 |
|
| 571 |
-
area =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 572 |
|
| 573 |
|
| 574 |
score = (
|
|
@@ -596,7 +656,7 @@ def detect_face(
|
|
| 596 |
|
| 597 |
|
| 598 |
# ============================================================
|
| 599 |
-
# INITIAL FACE
|
| 600 |
# ============================================================
|
| 601 |
|
| 602 |
def find_initial_face(
|
|
@@ -627,7 +687,7 @@ def find_initial_face(
|
|
| 627 |
cap.release()
|
| 628 |
|
| 629 |
raise RuntimeError(
|
| 630 |
-
"Video
|
| 631 |
)
|
| 632 |
|
| 633 |
|
|
@@ -687,7 +747,7 @@ def find_initial_face(
|
|
| 687 |
if not boxes:
|
| 688 |
|
| 689 |
raise RuntimeError(
|
| 690 |
-
"No face detected in
|
| 691 |
)
|
| 692 |
|
| 693 |
|
|
@@ -711,7 +771,7 @@ def find_initial_face(
|
|
| 711 |
|
| 712 |
|
| 713 |
# ============================================================
|
| 714 |
-
# SMOOTH FACE
|
| 715 |
# ============================================================
|
| 716 |
|
| 717 |
def smooth_face(
|
|
@@ -759,27 +819,20 @@ def smooth_face(
|
|
| 759 |
|
| 760 |
|
| 761 |
# ============================================================
|
| 762 |
-
#
|
| 763 |
# ============================================================
|
| 764 |
|
| 765 |
-
def
|
| 766 |
-
|
| 767 |
-
|
|
|
|
| 768 |
):
|
| 769 |
|
| 770 |
-
frame_h, frame_w = (
|
| 771 |
-
frame.shape[:2]
|
| 772 |
-
)
|
| 773 |
-
|
| 774 |
-
|
| 775 |
x, y, w, h = face_box
|
| 776 |
|
| 777 |
|
| 778 |
size = int(
|
| 779 |
-
max(
|
| 780 |
-
w,
|
| 781 |
-
h
|
| 782 |
-
)
|
| 783 |
*
|
| 784 |
(
|
| 785 |
1.0
|
|
@@ -789,69 +842,116 @@ def make_crop(
|
|
| 789 |
)
|
| 790 |
|
| 791 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 792 |
size = min(
|
| 793 |
size,
|
| 794 |
-
|
| 795 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 796 |
)
|
| 797 |
|
| 798 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 799 |
cx = (
|
| 800 |
-
|
| 801 |
+
|
| 802 |
-
|
| 803 |
)
|
| 804 |
|
| 805 |
|
| 806 |
cy = (
|
| 807 |
-
|
| 808 |
+
|
| 809 |
-
|
| 810 |
)
|
| 811 |
|
| 812 |
|
| 813 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 814 |
cx
|
| 815 |
-
|
| 816 |
-
|
| 817 |
)
|
| 818 |
|
| 819 |
|
| 820 |
-
|
| 821 |
cy
|
| 822 |
-
|
| 823 |
-
|
| 824 |
)
|
| 825 |
|
| 826 |
|
| 827 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
0,
|
| 829 |
min(
|
| 830 |
-
|
| 831 |
-
frame_w -
|
| 832 |
)
|
| 833 |
)
|
| 834 |
|
| 835 |
|
| 836 |
-
|
| 837 |
0,
|
| 838 |
min(
|
| 839 |
-
|
| 840 |
-
frame_h -
|
| 841 |
)
|
| 842 |
)
|
| 843 |
|
| 844 |
|
| 845 |
return (
|
| 846 |
-
int(
|
| 847 |
-
int(
|
| 848 |
-
int(
|
| 849 |
-
int(
|
| 850 |
)
|
| 851 |
|
| 852 |
|
| 853 |
# ============================================================
|
| 854 |
-
# WAV2LIP
|
| 855 |
# ============================================================
|
| 856 |
|
| 857 |
def run_wav2lip(
|
|
@@ -860,14 +960,7 @@ def run_wav2lip(
|
|
| 860 |
):
|
| 861 |
|
| 862 |
# ========================================================
|
| 863 |
-
#
|
| 864 |
-
#
|
| 865 |
-
# OpenCV:
|
| 866 |
-
# BGR
|
| 867 |
-
#
|
| 868 |
-
# Wav2Lip:
|
| 869 |
-
# RGB
|
| 870 |
-
#
|
| 871 |
# ========================================================
|
| 872 |
|
| 873 |
face_rgb = cv2.cvtColor(
|
|
@@ -903,11 +996,7 @@ def run_wav2lip(
|
|
| 903 |
# ========================================================
|
| 904 |
# 6 CHANNEL INPUT
|
| 905 |
#
|
| 906 |
-
# masked RGB
|
| 907 |
-
# +
|
| 908 |
-
# original RGB
|
| 909 |
-
#
|
| 910 |
-
# => 6 channels
|
| 911 |
# ========================================================
|
| 912 |
|
| 913 |
input_image = np.concatenate(
|
|
@@ -986,7 +1075,7 @@ def run_wav2lip(
|
|
| 986 |
|
| 987 |
|
| 988 |
# ========================================================
|
| 989 |
-
# OUTPUT
|
| 990 |
# ========================================================
|
| 991 |
|
| 992 |
if prediction.ndim == 4:
|
|
@@ -996,10 +1085,12 @@ def run_wav2lip(
|
|
| 996 |
|
| 997 |
if prediction.shape[0] == 3:
|
| 998 |
|
| 999 |
-
prediction =
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
|
|
|
|
|
|
|
| 1003 |
)
|
| 1004 |
|
| 1005 |
else:
|
|
@@ -1011,7 +1102,7 @@ def run_wav2lip(
|
|
| 1011 |
|
| 1012 |
|
| 1013 |
# ========================================================
|
| 1014 |
-
#
|
| 1015 |
# ========================================================
|
| 1016 |
|
| 1017 |
prediction = np.clip(
|
|
@@ -1058,20 +1149,42 @@ def create_mouth_mask(
|
|
| 1058 |
]
|
| 1059 |
|
| 1060 |
|
| 1061 |
-
#
|
| 1062 |
-
#
|
| 1063 |
-
#
|
| 1064 |
-
# Tight mask avoids coloring cheeks/nose.
|
| 1065 |
-
# ========================================================
|
| 1066 |
|
| 1067 |
-
cx =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1068 |
|
| 1069 |
-
cy = height * 0.715
|
| 1070 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1071 |
|
| 1072 |
-
rx = width * 0.29
|
| 1073 |
|
| 1074 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1075 |
|
| 1076 |
|
| 1077 |
distance = (
|
|
@@ -1094,21 +1207,25 @@ def create_mouth_mask(
|
|
| 1094 |
mask = np.exp(
|
| 1095 |
-distance
|
| 1096 |
*
|
| 1097 |
-
2.
|
| 1098 |
)
|
| 1099 |
|
| 1100 |
|
| 1101 |
-
#
|
| 1102 |
-
#
|
| 1103 |
-
#
|
| 1104 |
|
| 1105 |
top = int(
|
| 1106 |
-
height
|
|
|
|
|
|
|
| 1107 |
)
|
| 1108 |
|
| 1109 |
|
| 1110 |
bottom = int(
|
| 1111 |
-
height
|
|
|
|
|
|
|
| 1112 |
)
|
| 1113 |
|
| 1114 |
|
|
@@ -1124,28 +1241,26 @@ def create_mouth_mask(
|
|
| 1124 |
] = 0
|
| 1125 |
|
| 1126 |
|
| 1127 |
-
#
|
| 1128 |
-
#
|
| 1129 |
-
#
|
| 1130 |
-
# This preserves original lip boundary.
|
| 1131 |
-
# ========================================================
|
| 1132 |
|
| 1133 |
mask = np.clip(
|
| 1134 |
(
|
| 1135 |
mask
|
| 1136 |
-
|
| 1137 |
-
0.
|
| 1138 |
)
|
| 1139 |
/
|
| 1140 |
-
0.
|
| 1141 |
0,
|
| 1142 |
1
|
| 1143 |
)
|
| 1144 |
|
| 1145 |
|
| 1146 |
-
#
|
| 1147 |
-
#
|
| 1148 |
-
#
|
| 1149 |
|
| 1150 |
mask = (
|
| 1151 |
mask
|
|
@@ -1162,23 +1277,23 @@ def create_mouth_mask(
|
|
| 1162 |
)
|
| 1163 |
|
| 1164 |
|
| 1165 |
-
#
|
| 1166 |
-
#
|
| 1167 |
-
#
|
| 1168 |
|
| 1169 |
mask = cv2.GaussianBlur(
|
| 1170 |
mask.astype(
|
| 1171 |
np.float32
|
| 1172 |
),
|
| 1173 |
(0, 0),
|
| 1174 |
-
|
| 1175 |
)
|
| 1176 |
|
| 1177 |
|
| 1178 |
return np.clip(
|
| 1179 |
mask,
|
| 1180 |
-
0
|
| 1181 |
-
1
|
| 1182 |
)
|
| 1183 |
|
| 1184 |
|
|
@@ -1192,6 +1307,10 @@ def correct_generated_color(
|
|
| 1192 |
mask
|
| 1193 |
):
|
| 1194 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1195 |
original_lab = cv2.cvtColor(
|
| 1196 |
original_bgr,
|
| 1197 |
cv2.COLOR_BGR2LAB
|
|
@@ -1208,10 +1327,6 @@ def correct_generated_color(
|
|
| 1208 |
)
|
| 1209 |
|
| 1210 |
|
| 1211 |
-
# ========================================================
|
| 1212 |
-
# STRONG MOUTH AREA
|
| 1213 |
-
# ========================================================
|
| 1214 |
-
|
| 1215 |
strong = (
|
| 1216 |
mask > 0.45
|
| 1217 |
)
|
|
@@ -1224,9 +1339,9 @@ def correct_generated_color(
|
|
| 1224 |
return generated_bgr
|
| 1225 |
|
| 1226 |
|
| 1227 |
-
#
|
| 1228 |
# LUMINANCE
|
| 1229 |
-
#
|
| 1230 |
|
| 1231 |
original_l = np.median(
|
| 1232 |
original_lab[
|
|
@@ -1257,14 +1372,11 @@ def correct_generated_color(
|
|
| 1257 |
) * 0.30
|
| 1258 |
|
| 1259 |
|
| 1260 |
-
#
|
| 1261 |
-
# COLOR
|
| 1262 |
-
#
|
| 1263 |
-
# Small correction only.
|
| 1264 |
#
|
| 1265 |
-
#
|
| 1266 |
-
#
|
| 1267 |
-
# ========================================================
|
| 1268 |
|
| 1269 |
for channel in (
|
| 1270 |
1,
|
|
@@ -1307,10 +1419,6 @@ def correct_generated_color(
|
|
| 1307 |
)
|
| 1308 |
|
| 1309 |
|
| 1310 |
-
# ========================================================
|
| 1311 |
-
# CLAMP
|
| 1312 |
-
# ========================================================
|
| 1313 |
-
|
| 1314 |
generated_lab = np.clip(
|
| 1315 |
generated_lab,
|
| 1316 |
0,
|
|
@@ -1320,9 +1428,9 @@ def correct_generated_color(
|
|
| 1320 |
)
|
| 1321 |
|
| 1322 |
|
| 1323 |
-
#
|
| 1324 |
# LAB -> BGR
|
| 1325 |
-
#
|
| 1326 |
|
| 1327 |
corrected = cv2.cvtColor(
|
| 1328 |
generated_lab,
|
|
@@ -1334,7 +1442,7 @@ def correct_generated_color(
|
|
| 1334 |
|
| 1335 |
|
| 1336 |
# ============================================================
|
| 1337 |
-
#
|
| 1338 |
# ============================================================
|
| 1339 |
|
| 1340 |
def blend_face(
|
|
@@ -1347,20 +1455,23 @@ def blend_face(
|
|
| 1347 |
)
|
| 1348 |
|
| 1349 |
|
| 1350 |
-
#
|
| 1351 |
-
#
|
| 1352 |
-
#
|
| 1353 |
|
| 1354 |
generated_bgr = cv2.resize(
|
| 1355 |
generated_bgr,
|
| 1356 |
-
(
|
|
|
|
|
|
|
|
|
|
| 1357 |
interpolation=cv2.INTER_CUBIC
|
| 1358 |
)
|
| 1359 |
|
| 1360 |
|
| 1361 |
-
#
|
| 1362 |
-
#
|
| 1363 |
-
#
|
| 1364 |
|
| 1365 |
mask = create_mouth_mask(
|
| 1366 |
h,
|
|
@@ -1368,9 +1479,9 @@ def blend_face(
|
|
| 1368 |
)
|
| 1369 |
|
| 1370 |
|
| 1371 |
-
#
|
| 1372 |
-
#
|
| 1373 |
-
#
|
| 1374 |
|
| 1375 |
generated_bgr = (
|
| 1376 |
correct_generated_color(
|
|
@@ -1381,14 +1492,16 @@ def blend_face(
|
|
| 1381 |
)
|
| 1382 |
|
| 1383 |
|
| 1384 |
-
#
|
| 1385 |
-
#
|
| 1386 |
#
|
| 1387 |
-
#
|
| 1388 |
-
#
|
| 1389 |
|
| 1390 |
temporal_mask = np.clip(
|
| 1391 |
-
mask
|
|
|
|
|
|
|
| 1392 |
0,
|
| 1393 |
1
|
| 1394 |
)
|
|
@@ -1402,11 +1515,9 @@ def blend_face(
|
|
| 1402 |
)
|
| 1403 |
|
| 1404 |
|
| 1405 |
-
#
|
| 1406 |
-
#
|
| 1407 |
-
#
|
| 1408 |
-
# Keep original texture.
|
| 1409 |
-
# ========================================================
|
| 1410 |
|
| 1411 |
alpha = (
|
| 1412 |
mask
|
|
@@ -1451,7 +1562,7 @@ def blend_face(
|
|
| 1451 |
)
|
| 1452 |
|
| 1453 |
|
| 1454 |
-
|
| 1455 |
result,
|
| 1456 |
0,
|
| 1457 |
255
|
|
@@ -1460,6 +1571,9 @@ def blend_face(
|
|
| 1460 |
)
|
| 1461 |
|
| 1462 |
|
|
|
|
|
|
|
|
|
|
| 1463 |
# ============================================================
|
| 1464 |
# FFMPEG
|
| 1465 |
# ============================================================
|
|
@@ -1482,7 +1596,7 @@ def get_ffmpeg():
|
|
| 1482 |
|
| 1483 |
|
| 1484 |
# ============================================================
|
| 1485 |
-
#
|
| 1486 |
# ============================================================
|
| 1487 |
|
| 1488 |
def mux_audio(
|
|
@@ -1496,6 +1610,7 @@ def mux_audio(
|
|
| 1496 |
|
| 1497 |
command = [
|
| 1498 |
ffmpeg,
|
|
|
|
| 1499 |
"-y",
|
| 1500 |
|
| 1501 |
"-i",
|
|
@@ -1622,7 +1737,7 @@ def process_video(
|
|
| 1622 |
):
|
| 1623 |
|
| 1624 |
# ========================================================
|
| 1625 |
-
# RESET TEMPORAL
|
| 1626 |
# ========================================================
|
| 1627 |
|
| 1628 |
MOUTH_SMOOTHER.reset()
|
|
@@ -1738,7 +1853,7 @@ def process_video(
|
|
| 1738 |
|
| 1739 |
|
| 1740 |
# ====================================================
|
| 1741 |
-
# VIDEO
|
| 1742 |
# ====================================================
|
| 1743 |
|
| 1744 |
progress(
|
|
@@ -1811,7 +1926,7 @@ def process_video(
|
|
| 1811 |
|
| 1812 |
|
| 1813 |
# ====================================================
|
| 1814 |
-
# FACE
|
| 1815 |
# ====================================================
|
| 1816 |
|
| 1817 |
progress(
|
|
@@ -1833,6 +1948,32 @@ def process_video(
|
|
| 1833 |
)
|
| 1834 |
|
| 1835 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1836 |
# ====================================================
|
| 1837 |
# VIDEO WRITER
|
| 1838 |
# ====================================================
|
|
@@ -1865,7 +2006,7 @@ def process_video(
|
|
| 1865 |
|
| 1866 |
|
| 1867 |
# ====================================================
|
| 1868 |
-
#
|
| 1869 |
# ====================================================
|
| 1870 |
|
| 1871 |
audio_frame_count = int(
|
|
@@ -1884,9 +2025,7 @@ def process_video(
|
|
| 1884 |
|
| 1885 |
|
| 1886 |
# ====================================================
|
| 1887 |
-
#
|
| 1888 |
-
#
|
| 1889 |
-
# 80 mel frames / second
|
| 1890 |
# ====================================================
|
| 1891 |
|
| 1892 |
mel_idx_multiplier = (
|
|
@@ -1903,7 +2042,7 @@ def process_video(
|
|
| 1903 |
|
| 1904 |
|
| 1905 |
# ====================================================
|
| 1906 |
-
# FACE REDETECTION
|
| 1907 |
# ====================================================
|
| 1908 |
|
| 1909 |
redetect_every = max(
|
|
@@ -1917,7 +2056,7 @@ def process_video(
|
|
| 1917 |
|
| 1918 |
|
| 1919 |
# ====================================================
|
| 1920 |
-
# PROCESS
|
| 1921 |
# ====================================================
|
| 1922 |
|
| 1923 |
for frame_index in range(
|
|
@@ -1964,13 +2103,14 @@ def process_video(
|
|
| 1964 |
|
| 1965 |
|
| 1966 |
# =================================================
|
| 1967 |
-
#
|
| 1968 |
# =================================================
|
| 1969 |
|
| 1970 |
x, y, crop_w, crop_h = (
|
| 1971 |
-
|
| 1972 |
frame,
|
| 1973 |
-
current_face
|
|
|
|
| 1974 |
)
|
| 1975 |
)
|
| 1976 |
|
|
@@ -1991,7 +2131,7 @@ def process_video(
|
|
| 1991 |
|
| 1992 |
|
| 1993 |
# =================================================
|
| 1994 |
-
# MEL
|
| 1995 |
# =================================================
|
| 1996 |
|
| 1997 |
mel_start = int(
|
|
@@ -2029,12 +2169,16 @@ def process_video(
|
|
| 2029 |
|
| 2030 |
|
| 2031 |
# =================================================
|
| 2032 |
-
# EXACT
|
| 2033 |
# =================================================
|
| 2034 |
|
| 2035 |
if mel_chunk.shape[1] < 16:
|
| 2036 |
|
| 2037 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2038 |
|
| 2039 |
mel_chunk = np.zeros(
|
| 2040 |
(
|
|
@@ -2092,7 +2236,7 @@ def process_video(
|
|
| 2092 |
|
| 2093 |
|
| 2094 |
# =================================================
|
| 2095 |
-
#
|
| 2096 |
# =================================================
|
| 2097 |
|
| 2098 |
result_face = (
|
|
@@ -2165,7 +2309,7 @@ def process_video(
|
|
| 2165 |
|
| 2166 |
|
| 2167 |
# ====================================================
|
| 2168 |
-
# AUDIO
|
| 2169 |
# ====================================================
|
| 2170 |
|
| 2171 |
progress(
|
|
@@ -2182,7 +2326,7 @@ def process_video(
|
|
| 2182 |
|
| 2183 |
|
| 2184 |
# ====================================================
|
| 2185 |
-
#
|
| 2186 |
# ====================================================
|
| 2187 |
|
| 2188 |
progress(
|
|
@@ -2204,20 +2348,22 @@ def process_video(
|
|
| 2204 |
|
| 2205 |
status = (
|
| 2206 |
"## β
Lip Sync Completed\n\n"
|
| 2207 |
-
f"**
|
| 2208 |
f"`{width} Γ {height}`\n\n"
|
| 2209 |
f"**FPS:** "
|
| 2210 |
f"`{fps:.2f}`\n\n"
|
| 2211 |
-
f"**Audio:** "
|
| 2212 |
f"`{audio_duration:.2f}s`\n\n"
|
| 2213 |
-
f"**Frames:** "
|
| 2214 |
f"`{frames_to_process}`\n\n"
|
| 2215 |
-
"**
|
| 2216 |
-
"
|
| 2217 |
-
"**
|
| 2218 |
-
"**
|
| 2219 |
-
"**
|
|
|
|
| 2220 |
"**Face tracking:** β
\n\n"
|
|
|
|
| 2221 |
f"**Output size:** "
|
| 2222 |
f"`{output_size:.2f} MB`"
|
| 2223 |
)
|
|
@@ -2285,7 +2431,7 @@ CSS = """
|
|
| 2285 |
|
| 2286 |
|
| 2287 |
# ============================================================
|
| 2288 |
-
# GRADIO
|
| 2289 |
# ============================================================
|
| 2290 |
|
| 2291 |
with gr.Blocks(
|
|
@@ -2297,22 +2443,24 @@ with gr.Blocks(
|
|
| 2297 |
# π¬ Wav2Lip AI β High Quality Lip Sync
|
| 2298 |
|
| 2299 |
Upload a source video and target speech audio.
|
| 2300 |
-
The AI
|
| 2301 |
"""
|
| 2302 |
)
|
| 2303 |
|
| 2304 |
|
| 2305 |
gr.Markdown(
|
| 2306 |
"""
|
| 2307 |
-
###
|
| 2308 |
-
|
| 2309 |
-
-
|
| 2310 |
-
-
|
| 2311 |
-
- π
|
| 2312 |
-
-
|
| 2313 |
-
-
|
| 2314 |
-
-
|
|
|
|
| 2315 |
- π Accurate audio/mel timing
|
|
|
|
| 2316 |
"""
|
| 2317 |
)
|
| 2318 |
|
|
@@ -2370,23 +2518,22 @@ The AI synchronizes the visible mouth with the target audio.
|
|
| 2370 |
|
| 2371 |
gr.Markdown(
|
| 2372 |
"""
|
| 2373 |
-
###
|
| 2374 |
|
| 2375 |
-
|
| 2376 |
-
|
| 2377 |
-
|
| 2378 |
-
|
| 2379 |
-
|
| 2380 |
-
|
| 2381 |
|
| 2382 |
-
**
|
| 2383 |
-
The uploaded audio is what controls the lip movement.
|
| 2384 |
"""
|
| 2385 |
)
|
| 2386 |
|
| 2387 |
|
| 2388 |
# ========================================================
|
| 2389 |
-
#
|
| 2390 |
# ========================================================
|
| 2391 |
|
| 2392 |
generate_button.click(
|
|
|
|
| 14 |
|
| 15 |
|
| 16 |
# ============================================================
|
| 17 |
+
# CONFIG
|
| 18 |
# ============================================================
|
| 19 |
|
| 20 |
MODEL_PATH = "wav2lip_models/wav2lip_gan.onnx"
|
|
|
|
| 46 |
|
| 47 |
|
| 48 |
# ============================================================
|
| 49 |
+
# QUALITY SETTINGS
|
| 50 |
# ============================================================
|
| 51 |
|
| 52 |
+
# Lower value preserves original face texture.
|
| 53 |
+
# Higher value = stronger generated lips.
|
| 54 |
+
BLEND_STRENGTH = 0.50
|
|
|
|
| 55 |
|
| 56 |
+
# Temporal stabilization.
|
| 57 |
+
# 0.25-0.40 is good for natural lip movement.
|
| 58 |
+
TEMPORAL_ALPHA = 0.35
|
| 59 |
+
|
| 60 |
+
# Face tracking.
|
| 61 |
+
FACE_SMOOTHING = 0.15
|
| 62 |
+
|
| 63 |
+
# Re-detect face every N seconds.
|
| 64 |
+
REDETECT_SECONDS = 0.50
|
| 65 |
+
|
| 66 |
+
# Padding around detected face.
|
| 67 |
+
FACE_PADDING = 0.18
|
| 68 |
|
| 69 |
|
| 70 |
# ============================================================
|
| 71 |
+
# AUDIO SETTINGS
|
| 72 |
# ============================================================
|
| 73 |
|
| 74 |
SAMPLE_RATE = 16000
|
|
|
|
| 87 |
|
| 88 |
MEL_STEP_SIZE = 16
|
| 89 |
|
| 90 |
+
# Wav2Lip uses approximately 80 mel frames/sec.
|
| 91 |
MEL_FPS = 80.0
|
| 92 |
|
| 93 |
|
| 94 |
# ============================================================
|
| 95 |
+
# OUTPUT SETTINGS
|
| 96 |
# ============================================================
|
| 97 |
|
| 98 |
+
CRF = 17
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
+
PRESET = "medium"
|
|
|
|
|
|
|
| 101 |
|
| 102 |
|
| 103 |
# ============================================================
|
| 104 |
+
# DIRECTORIES
|
| 105 |
# ============================================================
|
| 106 |
|
| 107 |
+
os.makedirs(
|
| 108 |
+
OUTPUT_DIR,
|
| 109 |
+
exist_ok=True
|
| 110 |
+
)
|
| 111 |
|
| 112 |
+
os.makedirs(
|
| 113 |
+
FACE_DIR,
|
| 114 |
+
exist_ok=True
|
| 115 |
+
)
|
| 116 |
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
print("=" * 70)
|
| 119 |
print("Wav2Lip High Quality Lip Sync")
|
|
|
|
| 127 |
if not os.path.exists(MODEL_PATH):
|
| 128 |
|
| 129 |
raise FileNotFoundError(
|
| 130 |
+
"\n\n"
|
| 131 |
+
"Wav2Lip ONNX model not found:\n"
|
| 132 |
f"{MODEL_PATH}\n\n"
|
| 133 |
+
"Expected structure:\n"
|
| 134 |
"wav2lip_models/wav2lip_gan.onnx\n"
|
| 135 |
)
|
| 136 |
|
| 137 |
|
| 138 |
# ============================================================
|
| 139 |
+
# ONNX PROVIDERS
|
| 140 |
# ============================================================
|
| 141 |
|
| 142 |
available_providers = (
|
|
|
|
| 164 |
|
| 165 |
|
| 166 |
# ============================================================
|
| 167 |
+
# LOAD ONNX MODEL
|
| 168 |
# ============================================================
|
| 169 |
|
| 170 |
print(
|
| 171 |
"Loading Wav2Lip ONNX model..."
|
| 172 |
)
|
| 173 |
|
|
|
|
| 174 |
SESSION = ort.InferenceSession(
|
| 175 |
MODEL_PATH,
|
| 176 |
providers=ONNX_PROVIDERS
|
| 177 |
)
|
| 178 |
|
|
|
|
| 179 |
print(
|
| 180 |
"Model loaded successfully."
|
| 181 |
)
|
| 182 |
|
|
|
|
| 183 |
print(
|
| 184 |
"Model inputs:"
|
| 185 |
)
|
|
|
|
| 193 |
inp.type
|
| 194 |
)
|
| 195 |
|
|
|
|
| 196 |
print(
|
| 197 |
"Model outputs:"
|
| 198 |
)
|
|
|
|
| 208 |
|
| 209 |
|
| 210 |
# ============================================================
|
| 211 |
+
# DOWNLOAD FACE DETECTOR
|
| 212 |
# ============================================================
|
| 213 |
|
| 214 |
def download_file(url, path):
|
|
|
|
| 247 |
"Loading OpenCV DNN face detector..."
|
| 248 |
)
|
| 249 |
|
|
|
|
| 250 |
FACE_DETECTOR = cv2.dnn.readNetFromCaffe(
|
| 251 |
FACE_PROTO,
|
| 252 |
FACE_MODEL
|
| 253 |
)
|
| 254 |
|
|
|
|
| 255 |
print(
|
| 256 |
"Face detector loaded."
|
| 257 |
)
|
| 258 |
|
| 259 |
|
| 260 |
# ============================================================
|
| 261 |
+
# TEMPORAL SMOOTHER
|
| 262 |
# ============================================================
|
| 263 |
|
| 264 |
class MouthTemporalSmoother:
|
| 265 |
|
| 266 |
def __init__(
|
| 267 |
self,
|
| 268 |
+
alpha=0.35
|
| 269 |
):
|
| 270 |
|
| 271 |
self.alpha = alpha
|
| 272 |
|
| 273 |
self.previous = None
|
| 274 |
|
| 275 |
+
|
| 276 |
def reset(self):
|
| 277 |
|
| 278 |
self.previous = None
|
| 279 |
|
| 280 |
+
|
| 281 |
def smooth(
|
| 282 |
self,
|
| 283 |
generated,
|
| 284 |
mask
|
| 285 |
):
|
| 286 |
|
| 287 |
+
current = generated.astype(
|
| 288 |
+
np.float32
|
|
|
|
|
|
|
| 289 |
)
|
| 290 |
|
| 291 |
+
# ----------------------------------------------------
|
| 292 |
+
# First frame
|
| 293 |
+
# ----------------------------------------------------
|
| 294 |
+
|
| 295 |
if self.previous is None:
|
| 296 |
|
| 297 |
self.previous = current.copy()
|
|
|
|
| 299 |
return generated
|
| 300 |
|
| 301 |
|
| 302 |
+
# ----------------------------------------------------
|
| 303 |
+
# CRITICAL FIX
|
| 304 |
+
#
|
| 305 |
+
# Previous face crop can have different dimensions.
|
| 306 |
+
#
|
| 307 |
+
# Example:
|
| 308 |
+
#
|
| 309 |
+
# 1033 x 1033
|
| 310 |
+
# 1055 x 1055
|
| 311 |
+
#
|
| 312 |
+
# Resize previous frame before blending.
|
| 313 |
+
# ----------------------------------------------------
|
| 314 |
+
|
| 315 |
+
if (
|
| 316 |
+
self.previous.shape[:2]
|
| 317 |
+
!=
|
| 318 |
+
current.shape[:2]
|
| 319 |
+
):
|
| 320 |
+
|
| 321 |
+
self.previous = cv2.resize(
|
| 322 |
+
self.previous,
|
| 323 |
+
(
|
| 324 |
+
current.shape[1],
|
| 325 |
+
current.shape[0]
|
| 326 |
+
),
|
| 327 |
+
interpolation=cv2.INTER_LINEAR
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
self.previous = (
|
| 331 |
+
self.previous.astype(
|
| 332 |
+
np.float32
|
| 333 |
+
)
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
# ----------------------------------------------------
|
| 338 |
+
# Ensure mask matches current frame.
|
| 339 |
+
# ----------------------------------------------------
|
| 340 |
+
|
| 341 |
+
if (
|
| 342 |
+
mask.shape[:2]
|
| 343 |
+
!=
|
| 344 |
+
current.shape[:2]
|
| 345 |
+
):
|
| 346 |
+
|
| 347 |
+
mask = cv2.resize(
|
| 348 |
+
mask.astype(
|
| 349 |
+
np.float32
|
| 350 |
+
),
|
| 351 |
+
(
|
| 352 |
+
current.shape[1],
|
| 353 |
+
current.shape[0]
|
| 354 |
+
),
|
| 355 |
+
interpolation=cv2.INTER_LINEAR
|
| 356 |
+
)
|
| 357 |
|
| 358 |
|
| 359 |
# ----------------------------------------------------
|
| 360 |
+
# Temporal smoothing.
|
| 361 |
#
|
| 362 |
+
# This removes the "shadow moving over lips".
|
|
|
|
| 363 |
# ----------------------------------------------------
|
| 364 |
|
| 365 |
smoothed = (
|
| 366 |
+
self.previous
|
| 367 |
+
*
|
| 368 |
+
self.alpha
|
| 369 |
+
|
| 370 |
+
current
|
| 371 |
+
*
|
| 372 |
+
(
|
| 373 |
+
1.0
|
| 374 |
+
-
|
| 375 |
+
self.alpha
|
| 376 |
)
|
| 377 |
)
|
| 378 |
|
| 379 |
|
| 380 |
# ----------------------------------------------------
|
| 381 |
+
# Only stabilize mouth.
|
| 382 |
# ----------------------------------------------------
|
| 383 |
|
| 384 |
+
m = mask[..., None]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 385 |
|
| 386 |
|
| 387 |
result = (
|
|
|
|
| 393 |
)
|
| 394 |
|
| 395 |
|
| 396 |
+
result = np.clip(
|
|
|
|
|
|
|
|
|
|
| 397 |
result,
|
| 398 |
0,
|
| 399 |
255
|
|
|
|
| 402 |
)
|
| 403 |
|
| 404 |
|
| 405 |
+
self.previous = (
|
| 406 |
+
result.astype(
|
| 407 |
+
np.float32
|
| 408 |
+
)
|
| 409 |
+
)
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
return result
|
| 413 |
+
|
| 414 |
+
|
| 415 |
MOUTH_SMOOTHER = MouthTemporalSmoother(
|
| 416 |
+
alpha=TEMPORAL_ALPHA
|
| 417 |
)
|
| 418 |
|
| 419 |
|
| 420 |
# ============================================================
|
| 421 |
+
# LOAD AUDIO
|
| 422 |
# ============================================================
|
| 423 |
|
| 424 |
def load_audio(
|
|
|
|
| 437 |
"Could not load audio."
|
| 438 |
)
|
| 439 |
|
|
|
|
| 440 |
if len(audio) == 0:
|
| 441 |
|
| 442 |
raise RuntimeError(
|
| 443 |
"Audio file is empty."
|
| 444 |
)
|
| 445 |
|
|
|
|
| 446 |
audio = audio.astype(
|
| 447 |
np.float32
|
| 448 |
)
|
| 449 |
|
|
|
|
| 450 |
duration = (
|
| 451 |
len(audio)
|
| 452 |
/
|
| 453 |
SAMPLE_RATE
|
| 454 |
)
|
| 455 |
|
| 456 |
+
return (
|
| 457 |
+
audio,
|
| 458 |
+
duration
|
| 459 |
+
)
|
| 460 |
|
| 461 |
|
| 462 |
# ============================================================
|
| 463 |
+
# CREATE MEL
|
| 464 |
# ============================================================
|
| 465 |
|
| 466 |
def make_mel(
|
|
|
|
| 481 |
center=True
|
| 482 |
)
|
| 483 |
|
|
|
|
| 484 |
mel = np.maximum(
|
| 485 |
mel,
|
| 486 |
1e-5
|
| 487 |
)
|
| 488 |
|
|
|
|
| 489 |
mel = np.log10(
|
| 490 |
mel
|
| 491 |
)
|
| 492 |
|
|
|
|
| 493 |
return mel.astype(
|
| 494 |
np.float32
|
| 495 |
)
|
|
|
|
| 503 |
frame
|
| 504 |
):
|
| 505 |
|
| 506 |
+
frame_h, frame_w = (
|
| 507 |
+
frame.shape[:2]
|
| 508 |
+
)
|
| 509 |
|
| 510 |
|
| 511 |
blob = cv2.dnn.blobFromImage(
|
|
|
|
| 550 |
)
|
| 551 |
|
| 552 |
|
| 553 |
+
if confidence < 0.50:
|
| 554 |
|
| 555 |
continue
|
| 556 |
|
|
|
|
| 565 |
*
|
| 566 |
np.array(
|
| 567 |
[
|
| 568 |
+
frame_w,
|
| 569 |
+
frame_h,
|
| 570 |
+
frame_w,
|
| 571 |
+
frame_h
|
| 572 |
]
|
| 573 |
)
|
| 574 |
)
|
|
|
|
| 585 |
0,
|
| 586 |
min(
|
| 587 |
x1,
|
| 588 |
+
frame_w - 1
|
| 589 |
)
|
| 590 |
)
|
| 591 |
|
|
|
|
| 593 |
0,
|
| 594 |
min(
|
| 595 |
y1,
|
| 596 |
+
frame_h - 1
|
| 597 |
)
|
| 598 |
)
|
| 599 |
|
|
|
|
| 601 |
0,
|
| 602 |
min(
|
| 603 |
x2,
|
| 604 |
+
frame_w
|
| 605 |
)
|
| 606 |
)
|
| 607 |
|
|
|
|
| 609 |
0,
|
| 610 |
min(
|
| 611 |
y2,
|
| 612 |
+
frame_h
|
| 613 |
)
|
| 614 |
)
|
| 615 |
|
|
|
|
| 624 |
continue
|
| 625 |
|
| 626 |
|
| 627 |
+
area = (
|
| 628 |
+
bw
|
| 629 |
+
*
|
| 630 |
+
bh
|
| 631 |
+
)
|
| 632 |
|
| 633 |
|
| 634 |
score = (
|
|
|
|
| 656 |
|
| 657 |
|
| 658 |
# ============================================================
|
| 659 |
+
# FIND INITIAL FACE
|
| 660 |
# ============================================================
|
| 661 |
|
| 662 |
def find_initial_face(
|
|
|
|
| 687 |
cap.release()
|
| 688 |
|
| 689 |
raise RuntimeError(
|
| 690 |
+
"Video has no frames."
|
| 691 |
)
|
| 692 |
|
| 693 |
|
|
|
|
| 747 |
if not boxes:
|
| 748 |
|
| 749 |
raise RuntimeError(
|
| 750 |
+
"No face detected in video."
|
| 751 |
)
|
| 752 |
|
| 753 |
|
|
|
|
| 771 |
|
| 772 |
|
| 773 |
# ============================================================
|
| 774 |
+
# SMOOTH FACE POSITION
|
| 775 |
# ============================================================
|
| 776 |
|
| 777 |
def smooth_face(
|
|
|
|
| 819 |
|
| 820 |
|
| 821 |
# ============================================================
|
| 822 |
+
# FIXED CROP SIZE
|
| 823 |
# ============================================================
|
| 824 |
|
| 825 |
+
def create_fixed_crop_size(
|
| 826 |
+
face_box,
|
| 827 |
+
frame_width,
|
| 828 |
+
frame_height
|
| 829 |
):
|
| 830 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 831 |
x, y, w, h = face_box
|
| 832 |
|
| 833 |
|
| 834 |
size = int(
|
| 835 |
+
max(w, h)
|
|
|
|
|
|
|
|
|
|
| 836 |
*
|
| 837 |
(
|
| 838 |
1.0
|
|
|
|
| 842 |
)
|
| 843 |
|
| 844 |
|
| 845 |
+
size = max(
|
| 846 |
+
size,
|
| 847 |
+
256
|
| 848 |
+
)
|
| 849 |
+
|
| 850 |
+
|
| 851 |
size = min(
|
| 852 |
size,
|
| 853 |
+
frame_width,
|
| 854 |
+
frame_height
|
| 855 |
+
)
|
| 856 |
+
|
| 857 |
+
|
| 858 |
+
return int(
|
| 859 |
+
size
|
| 860 |
+
)
|
| 861 |
+
|
| 862 |
+
|
| 863 |
+
# ============================================================
|
| 864 |
+
# GET FIXED FACE CROP
|
| 865 |
+
# ============================================================
|
| 866 |
+
|
| 867 |
+
def get_fixed_face_crop(
|
| 868 |
+
frame,
|
| 869 |
+
face_box,
|
| 870 |
+
crop_size
|
| 871 |
+
):
|
| 872 |
+
|
| 873 |
+
frame_h, frame_w = (
|
| 874 |
+
frame.shape[:2]
|
| 875 |
+
)
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
x_face, y_face, w_face, h_face = (
|
| 879 |
+
face_box
|
| 880 |
)
|
| 881 |
|
| 882 |
|
| 883 |
+
# --------------------------------------------------------
|
| 884 |
+
# Face center
|
| 885 |
+
# --------------------------------------------------------
|
| 886 |
+
|
| 887 |
cx = (
|
| 888 |
+
x_face
|
| 889 |
+
|
| 890 |
+
w_face / 2.0
|
| 891 |
)
|
| 892 |
|
| 893 |
|
| 894 |
cy = (
|
| 895 |
+
y_face
|
| 896 |
+
|
| 897 |
+
h_face / 2.0
|
| 898 |
)
|
| 899 |
|
| 900 |
|
| 901 |
+
# Small upward correction.
|
| 902 |
+
cy -= h_face * 0.02
|
| 903 |
+
|
| 904 |
+
|
| 905 |
+
# --------------------------------------------------------
|
| 906 |
+
# Fixed-size crop.
|
| 907 |
+
# --------------------------------------------------------
|
| 908 |
+
|
| 909 |
+
x = int(
|
| 910 |
cx
|
| 911 |
-
|
| 912 |
+
crop_size / 2
|
| 913 |
)
|
| 914 |
|
| 915 |
|
| 916 |
+
y = int(
|
| 917 |
cy
|
| 918 |
-
|
| 919 |
+
crop_size / 2
|
| 920 |
)
|
| 921 |
|
| 922 |
|
| 923 |
+
# --------------------------------------------------------
|
| 924 |
+
# Keep inside frame.
|
| 925 |
+
# --------------------------------------------------------
|
| 926 |
+
|
| 927 |
+
x = max(
|
| 928 |
0,
|
| 929 |
min(
|
| 930 |
+
x,
|
| 931 |
+
frame_w - crop_size
|
| 932 |
)
|
| 933 |
)
|
| 934 |
|
| 935 |
|
| 936 |
+
y = max(
|
| 937 |
0,
|
| 938 |
min(
|
| 939 |
+
y,
|
| 940 |
+
frame_h - crop_size
|
| 941 |
)
|
| 942 |
)
|
| 943 |
|
| 944 |
|
| 945 |
return (
|
| 946 |
+
int(x),
|
| 947 |
+
int(y),
|
| 948 |
+
int(crop_size),
|
| 949 |
+
int(crop_size)
|
| 950 |
)
|
| 951 |
|
| 952 |
|
| 953 |
# ============================================================
|
| 954 |
+
# WAV2LIP INFERENCE
|
| 955 |
# ============================================================
|
| 956 |
|
| 957 |
def run_wav2lip(
|
|
|
|
| 960 |
):
|
| 961 |
|
| 962 |
# ========================================================
|
| 963 |
+
# BGR -> RGB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 964 |
# ========================================================
|
| 965 |
|
| 966 |
face_rgb = cv2.cvtColor(
|
|
|
|
| 996 |
# ========================================================
|
| 997 |
# 6 CHANNEL INPUT
|
| 998 |
#
|
| 999 |
+
# masked RGB + original RGB
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1000 |
# ========================================================
|
| 1001 |
|
| 1002 |
input_image = np.concatenate(
|
|
|
|
| 1075 |
|
| 1076 |
|
| 1077 |
# ========================================================
|
| 1078 |
+
# OUTPUT
|
| 1079 |
# ========================================================
|
| 1080 |
|
| 1081 |
if prediction.ndim == 4:
|
|
|
|
| 1085 |
|
| 1086 |
if prediction.shape[0] == 3:
|
| 1087 |
|
| 1088 |
+
prediction = (
|
| 1089 |
+
prediction.transpose(
|
| 1090 |
+
1,
|
| 1091 |
+
2,
|
| 1092 |
+
0
|
| 1093 |
+
)
|
| 1094 |
)
|
| 1095 |
|
| 1096 |
else:
|
|
|
|
| 1102 |
|
| 1103 |
|
| 1104 |
# ========================================================
|
| 1105 |
+
# 0-1 -> 0-255
|
| 1106 |
# ========================================================
|
| 1107 |
|
| 1108 |
prediction = np.clip(
|
|
|
|
| 1149 |
]
|
| 1150 |
|
| 1151 |
|
| 1152 |
+
# --------------------------------------------------------
|
| 1153 |
+
# Mouth center.
|
| 1154 |
+
# --------------------------------------------------------
|
|
|
|
|
|
|
| 1155 |
|
| 1156 |
+
cx = (
|
| 1157 |
+
width
|
| 1158 |
+
*
|
| 1159 |
+
0.50
|
| 1160 |
+
)
|
| 1161 |
|
|
|
|
| 1162 |
|
| 1163 |
+
cy = (
|
| 1164 |
+
height
|
| 1165 |
+
*
|
| 1166 |
+
0.715
|
| 1167 |
+
)
|
| 1168 |
|
|
|
|
| 1169 |
|
| 1170 |
+
# --------------------------------------------------------
|
| 1171 |
+
# Tighter mouth area.
|
| 1172 |
+
#
|
| 1173 |
+
# This is important for removing the moving shadow.
|
| 1174 |
+
# --------------------------------------------------------
|
| 1175 |
+
|
| 1176 |
+
rx = (
|
| 1177 |
+
width
|
| 1178 |
+
*
|
| 1179 |
+
0.25
|
| 1180 |
+
)
|
| 1181 |
+
|
| 1182 |
+
|
| 1183 |
+
ry = (
|
| 1184 |
+
height
|
| 1185 |
+
*
|
| 1186 |
+
0.13
|
| 1187 |
+
)
|
| 1188 |
|
| 1189 |
|
| 1190 |
distance = (
|
|
|
|
| 1207 |
mask = np.exp(
|
| 1208 |
-distance
|
| 1209 |
*
|
| 1210 |
+
2.6
|
| 1211 |
)
|
| 1212 |
|
| 1213 |
|
| 1214 |
+
# --------------------------------------------------------
|
| 1215 |
+
# Restrict mouth.
|
| 1216 |
+
# --------------------------------------------------------
|
| 1217 |
|
| 1218 |
top = int(
|
| 1219 |
+
height
|
| 1220 |
+
*
|
| 1221 |
+
0.57
|
| 1222 |
)
|
| 1223 |
|
| 1224 |
|
| 1225 |
bottom = int(
|
| 1226 |
+
height
|
| 1227 |
+
*
|
| 1228 |
+
0.87
|
| 1229 |
)
|
| 1230 |
|
| 1231 |
|
|
|
|
| 1241 |
] = 0
|
| 1242 |
|
| 1243 |
|
| 1244 |
+
# --------------------------------------------------------
|
| 1245 |
+
# Soft threshold.
|
| 1246 |
+
# --------------------------------------------------------
|
|
|
|
|
|
|
| 1247 |
|
| 1248 |
mask = np.clip(
|
| 1249 |
(
|
| 1250 |
mask
|
| 1251 |
-
|
| 1252 |
+
0.08
|
| 1253 |
)
|
| 1254 |
/
|
| 1255 |
+
0.92,
|
| 1256 |
0,
|
| 1257 |
1
|
| 1258 |
)
|
| 1259 |
|
| 1260 |
|
| 1261 |
+
# --------------------------------------------------------
|
| 1262 |
+
# Smoothstep.
|
| 1263 |
+
# --------------------------------------------------------
|
| 1264 |
|
| 1265 |
mask = (
|
| 1266 |
mask
|
|
|
|
| 1277 |
)
|
| 1278 |
|
| 1279 |
|
| 1280 |
+
# --------------------------------------------------------
|
| 1281 |
+
# Feather.
|
| 1282 |
+
# --------------------------------------------------------
|
| 1283 |
|
| 1284 |
mask = cv2.GaussianBlur(
|
| 1285 |
mask.astype(
|
| 1286 |
np.float32
|
| 1287 |
),
|
| 1288 |
(0, 0),
|
| 1289 |
+
2.5
|
| 1290 |
)
|
| 1291 |
|
| 1292 |
|
| 1293 |
return np.clip(
|
| 1294 |
mask,
|
| 1295 |
+
0,
|
| 1296 |
+
1
|
| 1297 |
)
|
| 1298 |
|
| 1299 |
|
|
|
|
| 1307 |
mask
|
| 1308 |
):
|
| 1309 |
|
| 1310 |
+
# --------------------------------------------------------
|
| 1311 |
+
# BGR -> LAB
|
| 1312 |
+
# --------------------------------------------------------
|
| 1313 |
+
|
| 1314 |
original_lab = cv2.cvtColor(
|
| 1315 |
original_bgr,
|
| 1316 |
cv2.COLOR_BGR2LAB
|
|
|
|
| 1327 |
)
|
| 1328 |
|
| 1329 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1330 |
strong = (
|
| 1331 |
mask > 0.45
|
| 1332 |
)
|
|
|
|
| 1339 |
return generated_bgr
|
| 1340 |
|
| 1341 |
|
| 1342 |
+
# --------------------------------------------------------
|
| 1343 |
# LUMINANCE
|
| 1344 |
+
# --------------------------------------------------------
|
| 1345 |
|
| 1346 |
original_l = np.median(
|
| 1347 |
original_lab[
|
|
|
|
| 1372 |
) * 0.30
|
| 1373 |
|
| 1374 |
|
| 1375 |
+
# --------------------------------------------------------
|
| 1376 |
+
# COLOR
|
|
|
|
|
|
|
| 1377 |
#
|
| 1378 |
+
# Prevent blue/purple mouth.
|
| 1379 |
+
# --------------------------------------------------------
|
|
|
|
| 1380 |
|
| 1381 |
for channel in (
|
| 1382 |
1,
|
|
|
|
| 1419 |
)
|
| 1420 |
|
| 1421 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1422 |
generated_lab = np.clip(
|
| 1423 |
generated_lab,
|
| 1424 |
0,
|
|
|
|
| 1428 |
)
|
| 1429 |
|
| 1430 |
|
| 1431 |
+
# --------------------------------------------------------
|
| 1432 |
# LAB -> BGR
|
| 1433 |
+
# --------------------------------------------------------
|
| 1434 |
|
| 1435 |
corrected = cv2.cvtColor(
|
| 1436 |
generated_lab,
|
|
|
|
| 1442 |
|
| 1443 |
|
| 1444 |
# ============================================================
|
| 1445 |
+
# FINAL FACE BLENDING
|
| 1446 |
# ============================================================
|
| 1447 |
|
| 1448 |
def blend_face(
|
|
|
|
| 1455 |
)
|
| 1456 |
|
| 1457 |
|
| 1458 |
+
# --------------------------------------------------------
|
| 1459 |
+
# Resize generated face.
|
| 1460 |
+
# --------------------------------------------------------
|
| 1461 |
|
| 1462 |
generated_bgr = cv2.resize(
|
| 1463 |
generated_bgr,
|
| 1464 |
+
(
|
| 1465 |
+
w,
|
| 1466 |
+
h
|
| 1467 |
+
),
|
| 1468 |
interpolation=cv2.INTER_CUBIC
|
| 1469 |
)
|
| 1470 |
|
| 1471 |
|
| 1472 |
+
# --------------------------------------------------------
|
| 1473 |
+
# Mouth mask.
|
| 1474 |
+
# --------------------------------------------------------
|
| 1475 |
|
| 1476 |
mask = create_mouth_mask(
|
| 1477 |
h,
|
|
|
|
| 1479 |
)
|
| 1480 |
|
| 1481 |
|
| 1482 |
+
# --------------------------------------------------------
|
| 1483 |
+
# Color correction.
|
| 1484 |
+
# --------------------------------------------------------
|
| 1485 |
|
| 1486 |
generated_bgr = (
|
| 1487 |
correct_generated_color(
|
|
|
|
| 1492 |
)
|
| 1493 |
|
| 1494 |
|
| 1495 |
+
# --------------------------------------------------------
|
| 1496 |
+
# Temporal stabilization.
|
| 1497 |
#
|
| 1498 |
+
# Only apply it to mouth region.
|
| 1499 |
+
# --------------------------------------------------------
|
| 1500 |
|
| 1501 |
temporal_mask = np.clip(
|
| 1502 |
+
mask
|
| 1503 |
+
*
|
| 1504 |
+
0.60,
|
| 1505 |
0,
|
| 1506 |
1
|
| 1507 |
)
|
|
|
|
| 1515 |
)
|
| 1516 |
|
| 1517 |
|
| 1518 |
+
# --------------------------------------------------------
|
| 1519 |
+
# Final blending.
|
| 1520 |
+
# --------------------------------------------------------
|
|
|
|
|
|
|
| 1521 |
|
| 1522 |
alpha = (
|
| 1523 |
mask
|
|
|
|
| 1562 |
)
|
| 1563 |
|
| 1564 |
|
| 1565 |
+
result = np.clip(
|
| 1566 |
result,
|
| 1567 |
0,
|
| 1568 |
255
|
|
|
|
| 1571 |
)
|
| 1572 |
|
| 1573 |
|
| 1574 |
+
return result
|
| 1575 |
+
|
| 1576 |
+
|
| 1577 |
# ============================================================
|
| 1578 |
# FFMPEG
|
| 1579 |
# ============================================================
|
|
|
|
| 1596 |
|
| 1597 |
|
| 1598 |
# ============================================================
|
| 1599 |
+
# ADD AUDIO
|
| 1600 |
# ============================================================
|
| 1601 |
|
| 1602 |
def mux_audio(
|
|
|
|
| 1610 |
|
| 1611 |
command = [
|
| 1612 |
ffmpeg,
|
| 1613 |
+
|
| 1614 |
"-y",
|
| 1615 |
|
| 1616 |
"-i",
|
|
|
|
| 1737 |
):
|
| 1738 |
|
| 1739 |
# ========================================================
|
| 1740 |
+
# RESET TEMPORAL MEMORY
|
| 1741 |
# ========================================================
|
| 1742 |
|
| 1743 |
MOUTH_SMOOTHER.reset()
|
|
|
|
| 1853 |
|
| 1854 |
|
| 1855 |
# ====================================================
|
| 1856 |
+
# OPEN VIDEO
|
| 1857 |
# ====================================================
|
| 1858 |
|
| 1859 |
progress(
|
|
|
|
| 1926 |
|
| 1927 |
|
| 1928 |
# ====================================================
|
| 1929 |
+
# FIND FACE
|
| 1930 |
# ====================================================
|
| 1931 |
|
| 1932 |
progress(
|
|
|
|
| 1948 |
)
|
| 1949 |
|
| 1950 |
|
| 1951 |
+
# ====================================================
|
| 1952 |
+
# FIX CROP SIZE
|
| 1953 |
+
#
|
| 1954 |
+
# THIS PREVENTS:
|
| 1955 |
+
#
|
| 1956 |
+
# 1033x1033
|
| 1957 |
+
# 1055x1055
|
| 1958 |
+
#
|
| 1959 |
+
# broadcasting error.
|
| 1960 |
+
# ====================================================
|
| 1961 |
+
|
| 1962 |
+
fixed_crop_size = (
|
| 1963 |
+
create_fixed_crop_size(
|
| 1964 |
+
current_face,
|
| 1965 |
+
width,
|
| 1966 |
+
height
|
| 1967 |
+
)
|
| 1968 |
+
)
|
| 1969 |
+
|
| 1970 |
+
|
| 1971 |
+
print(
|
| 1972 |
+
"Fixed face crop size:",
|
| 1973 |
+
fixed_crop_size
|
| 1974 |
+
)
|
| 1975 |
+
|
| 1976 |
+
|
| 1977 |
# ====================================================
|
| 1978 |
# VIDEO WRITER
|
| 1979 |
# ====================================================
|
|
|
|
| 2006 |
|
| 2007 |
|
| 2008 |
# ====================================================
|
| 2009 |
+
# PROCESS ONLY AUDIO DURATION
|
| 2010 |
# ====================================================
|
| 2011 |
|
| 2012 |
audio_frame_count = int(
|
|
|
|
| 2025 |
|
| 2026 |
|
| 2027 |
# ====================================================
|
| 2028 |
+
# AUDIO -> VIDEO FRAME TIMING
|
|
|
|
|
|
|
| 2029 |
# ====================================================
|
| 2030 |
|
| 2031 |
mel_idx_multiplier = (
|
|
|
|
| 2042 |
|
| 2043 |
|
| 2044 |
# ====================================================
|
| 2045 |
+
# FACE REDETECTION INTERVAL
|
| 2046 |
# ====================================================
|
| 2047 |
|
| 2048 |
redetect_every = max(
|
|
|
|
| 2056 |
|
| 2057 |
|
| 2058 |
# ====================================================
|
| 2059 |
+
# PROCESS
|
| 2060 |
# ====================================================
|
| 2061 |
|
| 2062 |
for frame_index in range(
|
|
|
|
| 2103 |
|
| 2104 |
|
| 2105 |
# =================================================
|
| 2106 |
+
# FIXED-SIZE CROP
|
| 2107 |
# =================================================
|
| 2108 |
|
| 2109 |
x, y, crop_w, crop_h = (
|
| 2110 |
+
get_fixed_face_crop(
|
| 2111 |
frame,
|
| 2112 |
+
current_face,
|
| 2113 |
+
fixed_crop_size
|
| 2114 |
)
|
| 2115 |
)
|
| 2116 |
|
|
|
|
| 2131 |
|
| 2132 |
|
| 2133 |
# =================================================
|
| 2134 |
+
# MEL INDEX
|
| 2135 |
# =================================================
|
| 2136 |
|
| 2137 |
mel_start = int(
|
|
|
|
| 2169 |
|
| 2170 |
|
| 2171 |
# =================================================
|
| 2172 |
+
# EXACT 80x16
|
| 2173 |
# =================================================
|
| 2174 |
|
| 2175 |
if mel_chunk.shape[1] < 16:
|
| 2176 |
|
| 2177 |
+
if (
|
| 2178 |
+
mel_chunk.shape[1]
|
| 2179 |
+
==
|
| 2180 |
+
0
|
| 2181 |
+
):
|
| 2182 |
|
| 2183 |
mel_chunk = np.zeros(
|
| 2184 |
(
|
|
|
|
| 2236 |
|
| 2237 |
|
| 2238 |
# =================================================
|
| 2239 |
+
# QUALITY BLEND
|
| 2240 |
# =================================================
|
| 2241 |
|
| 2242 |
result_face = (
|
|
|
|
| 2309 |
|
| 2310 |
|
| 2311 |
# ====================================================
|
| 2312 |
+
# ADD AUDIO
|
| 2313 |
# ====================================================
|
| 2314 |
|
| 2315 |
progress(
|
|
|
|
| 2326 |
|
| 2327 |
|
| 2328 |
# ====================================================
|
| 2329 |
+
# FINISHED
|
| 2330 |
# ====================================================
|
| 2331 |
|
| 2332 |
progress(
|
|
|
|
| 2348 |
|
| 2349 |
status = (
|
| 2350 |
"## β
Lip Sync Completed\n\n"
|
| 2351 |
+
f"**Source resolution:** "
|
| 2352 |
f"`{width} Γ {height}`\n\n"
|
| 2353 |
f"**FPS:** "
|
| 2354 |
f"`{fps:.2f}`\n\n"
|
| 2355 |
+
f"**Audio duration:** "
|
| 2356 |
f"`{audio_duration:.2f}s`\n\n"
|
| 2357 |
+
f"**Frames processed:** "
|
| 2358 |
f"`{frames_to_process}`\n\n"
|
| 2359 |
+
f"**Fixed face crop:** "
|
| 2360 |
+
f"`{fixed_crop_size} Γ {fixed_crop_size}`\n\n"
|
| 2361 |
+
"**Lip color correction:** β
\n\n"
|
| 2362 |
+
"**Blue-mouth protection:** β
\n\n"
|
| 2363 |
+
"**Mouth shadow reduction:** β
\n\n"
|
| 2364 |
+
"**Temporal stabilization:** β
\n\n"
|
| 2365 |
"**Face tracking:** β
\n\n"
|
| 2366 |
+
"**Original skin texture:** preserved\n\n"
|
| 2367 |
f"**Output size:** "
|
| 2368 |
f"`{output_size:.2f} MB`"
|
| 2369 |
)
|
|
|
|
| 2431 |
|
| 2432 |
|
| 2433 |
# ============================================================
|
| 2434 |
+
# GRADIO APP
|
| 2435 |
# ============================================================
|
| 2436 |
|
| 2437 |
with gr.Blocks(
|
|
|
|
| 2443 |
# π¬ Wav2Lip AI β High Quality Lip Sync
|
| 2444 |
|
| 2445 |
Upload a source video and target speech audio.
|
| 2446 |
+
The AI will synchronize the mouth with the target audio.
|
| 2447 |
"""
|
| 2448 |
)
|
| 2449 |
|
| 2450 |
|
| 2451 |
gr.Markdown(
|
| 2452 |
"""
|
| 2453 |
+
### β¨ Processing improvements
|
| 2454 |
+
|
| 2455 |
+
- π More natural visible lips
|
| 2456 |
+
- ποΈ Temporal mouth stabilization
|
| 2457 |
+
- π Blue/purple mouth correction
|
| 2458 |
+
- π§΄ Reduced lip shadow/flicker
|
| 2459 |
+
- π― Tight mouth mask
|
| 2460 |
+
- π¨ LAB color matching
|
| 2461 |
+
- π₯ Stable face tracking
|
| 2462 |
- π Accurate audio/mel timing
|
| 2463 |
+
- π Original skin texture preserved
|
| 2464 |
"""
|
| 2465 |
)
|
| 2466 |
|
|
|
|
| 2518 |
|
| 2519 |
gr.Markdown(
|
| 2520 |
"""
|
| 2521 |
+
### π Best results
|
| 2522 |
|
| 2523 |
+
- Face looking toward camera
|
| 2524 |
+
- Lips clearly visible
|
| 2525 |
+
- Good lighting
|
| 2526 |
+
- Minimal motion blur
|
| 2527 |
+
- Clear speech
|
| 2528 |
+
- 720p or higher source
|
| 2529 |
|
| 2530 |
+
The **audio controls the lip movement**. The optional text is only displayed as a reference.
|
|
|
|
| 2531 |
"""
|
| 2532 |
)
|
| 2533 |
|
| 2534 |
|
| 2535 |
# ========================================================
|
| 2536 |
+
# BUTTON
|
| 2537 |
# ========================================================
|
| 2538 |
|
| 2539 |
generate_button.click(
|