File size: 52,441 Bytes
f80a40e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 | import numpy as np
import cv2, os, sys, subprocess, platform, torch
import math
from tqdm import tqdm
from PIL import Image
from scipy.io import loadmat
sys.path.insert(0, 'third_part')
sys.path.insert(0, 'third_part/GPEN')
sys.path.insert(0, 'third_part/GFPGAN')
# 3dmm extraction
from third_part.face3d.util.preprocess import align_img
from third_part.face3d.util.load_mats import load_lm3d
from third_part.face3d.extract_kp_videos import KeypointExtractor
# face enhancement
from third_part.GPEN.gpen_face_enhancer import FaceEnhancement
from third_part.GFPGAN.gfpgan import GFPGANer
# expression control
from third_part.ganimation_replicate.model.ganimation import GANimationModel
from utils import audio
from utils.ffhq_preprocess import Croper
from utils.alignment_stit import crop_faces, calc_alignment_coefficients, paste_image
from utils.inference_utils import Laplacian_Pyramid_Blending_with_mask, face_detect, load_model, options, split_coeff, \
trans_image, transform_semantic, find_crop_norm_ratio, load_face3d_net, exp_aus_dict, mask_postprocess
import warnings
warnings.filterwarnings("ignore")
args = options()
# Optional: attempt to load a SyncNet model if provided; otherwise fall back to proxy confidence
_syncnet = None
_syncnet_warned = False
def _maybe_load_syncnet(path):
global _syncnet, _syncnet_warned
if _syncnet is not None:
return _syncnet
try:
if os.path.isfile(path):
# Expect a TorchScript or torch.load-able module that exposes forward(audio, video)->embeds or confidence
_syncnet = torch.jit.load(path, map_location='cpu').eval()
else:
if not _syncnet_warned:
print(f"[SyncNet] Model not found at {path}. Using proxy confidence.")
_syncnet_warned = True
_syncnet = None
except Exception as e:
if not _syncnet_warned:
print(f"[SyncNet] Failed to load model ({e}). Using proxy confidence.")
_syncnet_warned = True
_syncnet = None
return _syncnet
def _ffprobe_duration(path):
try:
cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nk=1:nw=1', path]
out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return float(out.decode().strip())
except Exception as e:
print(f"[Warn] ffprobe failed for {path}: {e}")
return None
def _split_into_chunks(face_path, audio_path, out_dir, fps, chunk_duration, overlap, enforce_sr=16000, crossfade=False):
os.makedirs(out_dir, exist_ok=True)
video_dur = _ffprobe_duration(face_path) or 0
audio_dur = _ffprobe_duration(audio_path) or 0
total_dur = min(video_dur, audio_dur) if (video_dur > 0 and audio_dur > 0) else max(video_dur, audio_dur)
if total_dur == 0:
raise RuntimeError('Could not determine media duration')
eff_overlap = overlap if crossfade else 0.0
step = max(0.0, chunk_duration - eff_overlap) if chunk_duration > 0 else total_dur
starts = []
t = 0.0
while t < total_dur:
starts.append(t)
t += step if step > 0 else total_dur
chunk_paths = []
for idx, start in enumerate(starts):
dur = min(chunk_duration if chunk_duration > 0 else total_dur, total_dur - start)
v_out = os.path.join(out_dir, f'video_{idx+1:03d}.mp4')
a_out = os.path.join(out_dir, f'audio_{idx+1:03d}.wav')
# Re-encode video chunk to enforce exact boundaries & fps
vcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', face_path, '-t', f'{dur:.3f}',
'-r', f'{fps:.3f}', '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', '-pix_fmt', 'yuv420p', v_out]
acmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', audio_path, '-t', f'{dur:.3f}',
'-ac', '1', '-ar', str(enforce_sr), a_out]
try:
subprocess.check_call(vcmd, shell=False)
subprocess.check_call(acmd, shell=False)
except subprocess.CalledProcessError as e:
print(f"[Warn] Failed to split chunk {idx+1}: {e}")
continue
# Validate audio sample rate
try:
sr = _probe_audio_sr(a_out)
if sr != enforce_sr:
print(f"[Warn] Fixing sample rate for {a_out} (got {sr}); forcing {enforce_sr}")
fixcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', a_out, '-ac', '1', '-ar', str(enforce_sr), a_out]
subprocess.check_call(fixcmd, shell=False)
except Exception as e:
print(f"[Warn] Could not validate audio sr for {a_out}: {e}")
chunk_paths.append((v_out, a_out))
return chunk_paths
def _probe_audio_sr(path):
cmd = ['ffprobe', '-v', 'error', '-select_streams', 'a:0', '-show_entries', 'stream=sample_rate', '-of', 'default=nk=1:nw=1', path]
out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return int(out.decode().strip())
def _concat_media(list_file, output_path, stream='video', reencode=False, drop_audio=False):
# stream: 'video' or 'audio'
if stream == 'video':
if reencode:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c:v', 'libx264', '-pix_fmt', 'yuv420p']
else:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy']
if drop_audio:
cmd += ['-an']
cmd += [output_path]
else:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy', output_path]
subprocess.check_call(cmd, shell=False)
def _xfade_concatenate(videos, out_path, overlap):
# Simple pairwise xfade across N segments
if len(videos) == 1:
return videos[0]
cur = videos[0]
for i in range(1, len(videos)):
nxt = videos[i]
tmp = out_path.replace('.mp4', f'_xf{i:03d}.mp4')
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', cur, '-i', nxt,
'-filter_complex', f"[0:v][1:v]xfade=transition=fade:duration={overlap:.3f}:offset=PTS-STARTPTS[v];[0:a][1:a]acrossfade=d={overlap:.3f}[a]",
'-map', '[v]', '-map', '[a]', tmp]
subprocess.check_call(cmd, shell=False)
cur = tmp
os.replace(cur, out_path)
return out_path
# --- Video stabilization helpers ---
def _stabilize_video_with_vidstab(input_path, out_dir, shakiness=5, smoothing=30, crop_mode='keep'):
os.makedirs(out_dir, exist_ok=True)
trf_path = os.path.join(out_dir, 'vidstab_transform.trf')
stabilized_path = os.path.join(out_dir, 'stabilized_input.mp4')
# Normalize paths for ffmpeg filters
def _norm(p):
return os.path.abspath(p).replace('\\', '/')
input_norm = _norm(input_path)
trf_norm = _norm(trf_path)
# Pass 1: detect
detect_filter = f"vidstabdetect=shakiness={int(shakiness)}:accuracy=15:result={trf_norm}"
detect_cmd = ['ffmpeg', '-loglevel', 'error', '-y',
'-i', input_norm,
'-vf', detect_filter,
'-f', 'null', '-']
# Pass 2: transform
crop_mode = crop_mode if crop_mode in ['keep', 'content', 'black'] else 'keep'
transform_filter = f"vidstabtransform=smoothing={int(smoothing)}:input={trf_norm}:crop={crop_mode}"
transform_cmd = ['ffmpeg', '-loglevel', 'error', '-y',
'-i', input_norm,
'-vf', transform_filter,
'-c:v', 'libx264', '-preset', 'medium', '-crf', '18',
'-c:a', 'copy',
_norm(stabilized_path)]
try:
subprocess.check_call(detect_cmd, shell=False)
subprocess.check_call(transform_cmd, shell=False)
except subprocess.CalledProcessError as e:
raise RuntimeError(f'ffmpeg vidstab failed: {e}')
return stabilized_path
# --- Sync gating helpers ---
def _mouth_open_metric(lms):
try:
if lms is None or len(lms) < 68:
return 0.0
p = lms
pairs = [(62,66), (63,67), (61,65)]
vals = []
for a, b in pairs:
ya, yb = float(p[a][1]), float(p[b][1])
vals.append(abs(yb - ya))
return float(np.mean(vals)) if len(vals) > 0 else 0.0
except Exception:
return 0.0
def _sync_confidence(frame_idx, lm_all, mel_chunks, win=5):
try:
n = len(mel_chunks)
if n == 0:
return 0.0
s = max(0, frame_idx - win//2)
e = min(n, frame_idx + (win - win//2))
if e - s < 3:
return 0.0
mouth_series = np.array([_mouth_open_metric(lm_all[k]) for k in range(s, e)], dtype=np.float32)
energy_series = np.array([float((mel_chunks[k].astype(np.float32) ** 2).mean()) for k in range(s, e)], dtype=np.float32)
# normalize
def nz_norm(x):
x = x - x.mean()
sdev = x.std()
return x / (sdev if sdev > 1e-6 else 1.0)
m_norm = nz_norm(mouth_series)
e_norm = nz_norm(energy_series)
L = min(len(m_norm), len(e_norm))
if L < 3:
return 0.0
corr = float(np.corrcoef(m_norm[:L], e_norm[:L])[0, 1])
return max(0.0, min(1.0, 0.5 * (corr + 1.0)))
except Exception:
return 0.0
def main():
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('[Info] Using {} for inference.'.format(device))
os.makedirs(os.path.join('temp', args.tmp_dir), exist_ok=True)
enhancer = FaceEnhancement(base_dir='checkpoints', size=args.gpen_size, model=args.gpen_model, use_sr=False, \
sr_model='rrdb_realesrnet_psnr', channel_multiplier=2, narrow=1, device=device)
restorer = GFPGANer(model_path='checkpoints/GFPGANv1.3.pth', upscale=1, arch='clean', \
channel_multiplier=2, bg_upsampler=None)
base_name = args.face.split('/')[-1]
if os.path.isfile(args.face) and args.face.split('.')[1] in ['jpg', 'png', 'jpeg']:
args.static = True
# Enforce one-shot for static images
if not args.one_shot:
print('[Info] Static image detected; enabling one-shot mode.')
args.one_shot = True
if not os.path.isfile(args.face):
raise ValueError('--face argument must be a valid path to video/image file')
elif args.face.split('.')[1] in ['jpg', 'png', 'jpeg']:
full_frames = [cv2.imread(args.face)]
fps = args.fps
else:
# Disable one-shot for videos
if args.one_shot:
print('[Info] Video detected; disabling one-shot mode (only for static images).')
args.one_shot = False
if getattr(args, 'pre_stabilize_video', False):
try:
stabilize_dir = os.path.join('temp', args.tmp_dir, 'stabilize')
stabilized_face = _stabilize_video_with_vidstab(
args.face,
stabilize_dir,
shakiness=getattr(args, 'stabilize_shakiness', 5),
smoothing=getattr(args, 'stabilize_smoothing', 30),
crop_mode=getattr(args, 'stabilize_crop', 'keep')
)
print(f"[Stabilize] Using stabilized video at {stabilized_face}")
args.face = stabilized_face
except Exception as e:
print(f"[Stabilize] Warning: stabilization failed ({e}). Proceeding with original video.")
# Chunked processing for long videos
if args.enable_chunking and args.chunk_duration and args.chunk_duration > 0:
try:
video_dur = _ffprobe_duration(args.face) or 0
if video_dur > args.chunk_duration + 0.5:
# Probe fps
cap_tmp = cv2.VideoCapture(args.face)
fps_probe = cap_tmp.get(cv2.CAP_PROP_FPS)
cap_tmp.release()
if not fps_probe or math.isclose(fps_probe, 0.0) or np.isnan(fps_probe):
fps_probe = args.fps
parts_dir = os.path.join('temp', args.tmp_dir, args.chunk_temp_dir)
out_parts_dir = os.path.join('temp', args.tmp_dir, 'out_parts')
os.makedirs(parts_dir, exist_ok=True)
os.makedirs(out_parts_dir, exist_ok=True)
print(f"[Chunk] Splitting into ~{args.chunk_duration}s chunks with overlap={args.chunk_overlap}s")
chunks = _split_into_chunks(args.face, args.audio, parts_dir, fps_probe, args.chunk_duration, args.chunk_overlap, enforce_sr=16000, crossfade=args.chunk_crossfade)
# Minimal required seconds for a mel window
from utils import audio as _audio
min_sec = (args.mel_step_size * _audio.hp.hop_size) / _audio.hp.sample_rate
successes = []
for ci, (vpath, apath) in enumerate(chunks, start=1):
try:
# Validate length
c_dur = _ffprobe_duration(vpath) or 0
if c_dur < min_sec:
print(f"[Warn] Skipping chunk {ci:03d} too short: {c_dur:.3f}s < {min_sec:.3f}s")
continue
out_path = os.path.join(out_parts_dir, f'output_{ci:03d}.mp4')
cmd = [sys.executable, os.path.abspath(__file__), '--face', vpath, '--audio', apath, '--outfile', out_path, '--fps', f'{fps_probe:.3f}', '--tmp_dir', f"{args.tmp_dir}/chunk_{ci:03d}", '--chunk_duration', '0']
# Propagate useful flags
if args.audio_preprocess: cmd.append('--audio_preprocess')
if args.audio_normalize: cmd.append('--audio_normalize')
if args.audio_denoise: cmd.append('--audio_denoise')
if args.lip_temporal_smooth: cmd.append('--lip_temporal_smooth')
# Always re-preprocess per chunk to avoid stale landmark/cached files mismatch
cmd.append('--re_preprocess')
try:
subprocess.check_call(cmd, shell=False)
successes.append(out_path)
except subprocess.CalledProcessError as e:
print(f"[Warn] Chunk {ci:03d} failed: {e}")
continue
except Exception as e:
print(f"[Warn] Exception on chunk {ci:03d}: {e}")
continue
if len(successes) == 0:
raise RuntimeError('No chunk processed successfully')
final_out = args.outfile
if args.chunk_crossfade:
# Build crossfaded concatenation including audio
concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4')
_xfade_concatenate(successes, concat_video, max(0.0, args.chunk_overlap))
# Just move to final (already has audio)
if not os.path.isdir(os.path.dirname(final_out)):
os.makedirs(os.path.dirname(final_out), exist_ok=True)
if platform.system() == 'Windows':
subprocess.check_call(['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-c', 'copy', final_out], shell=False)
else:
subprocess.check_call(f'ffmpeg -loglevel error -y -i "{concat_video}" -c copy "{final_out}"', shell=True)
else:
# Concatenate videos (drop audio)
concat_v_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_v.txt')
with open(concat_v_list, 'w', encoding='utf-8') as f:
for p in successes:
_safe = os.path.abspath(p).replace('\\', '/')
f.write(f"file '{_safe}'\n")
concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4')
_concat_media(concat_v_list, concat_video, stream='video', reencode=False, drop_audio=True)
# Concatenate audios
chunk_audios = [os.path.join(parts_dir, f) for f in sorted(os.listdir(parts_dir)) if f.lower().startswith('audio_') and f.lower().endswith('.wav')]
concat_a_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_a.txt')
with open(concat_a_list, 'w', encoding='utf-8') as f:
for p in chunk_audios:
_safe = os.path.abspath(p).replace('\\', '/')
f.write(f"file '{_safe}'\n")
concat_audio = os.path.join('temp', args.tmp_dir, 'concat_audio.wav')
_concat_media(concat_a_list, concat_audio, stream='audio', reencode=False)
# Merge A/V with -shortest
if not os.path.isdir(os.path.dirname(final_out)):
os.makedirs(os.path.dirname(final_out), exist_ok=True)
if platform.system() == 'Windows':
mcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-i', concat_audio, '-shortest', '-c:v', 'copy', final_out]
subprocess.call(mcmd, shell=False)
else:
mcmd = f'ffmpeg -loglevel error -y -i "{concat_video}" -i "{concat_audio}" -shortest -c:v copy "{final_out}"'
subprocess.call(mcmd, shell=True)
print('outfile:', final_out)
return
except Exception as e:
print(f"[Warn] Chunked processing fallback due to error: {e}")
video_stream = cv2.VideoCapture(args.face)
fps = video_stream.get(cv2.CAP_PROP_FPS)
full_frames = []
while True:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
y1, y2, x1, x2 = args.crop
if x2 == -1: x2 = frame.shape[1]
if y2 == -1: y2 = frame.shape[0]
frame = frame[y1:y2, x1:x2]
full_frames.append(frame)
print ("[Step 0] Number of frames available for inference: "+str(len(full_frames)))
# face detection & cropping, cropping the first frame as the style of FFHQ
croper = Croper('checkpoints/shape_predictor_68_face_landmarks.dat')
full_frames_RGB = [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames]
full_frames_RGB, crop, quad = croper.crop(full_frames_RGB, xsize=512)
clx, cly, crx, cry = crop
lx, ly, rx, ry = quad
lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
oy1, oy2, ox1, ox2 = cly+ly, min(cly+ry, full_frames[0].shape[0]), clx+lx, min(clx+rx, full_frames[0].shape[1])
# original_size = (ox2 - ox1, oy2 - oy1)
frames_pil = [Image.fromarray(cv2.resize(frame,(256,256))) for frame in full_frames_RGB]
# get the landmark according to the detected face.
if not os.path.isfile('temp/'+base_name+'_landmarks.txt') or args.re_preprocess:
print('[Step 1] Landmarks Extraction in Video.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
print('[Step 1] Using saved landmarks.')
lm_path = 'temp/'+base_name+'_landmarks.txt'
try:
lm_raw = np.loadtxt(lm_path).astype(np.float32)
except Exception as e:
print(f"[Step 1] Failed to load saved landmarks ({e}); re-extracting.")
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
total_vals = lm_raw.size
denom = 2 * len(full_frames)
if denom > 0 and total_vals % denom == 0:
k = total_vals // denom
try:
lm = lm_raw.reshape([len(full_frames), k, 2])
except Exception:
print('[Step 1] Saved landmarks shape mismatch; re-extracting.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
print('[Step 1] Saved landmarks length does not match frames; re-extracting.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
if not os.path.isfile('temp/'+base_name+'_coeffs.npy') or args.exp_img is not None or args.re_preprocess:
net_recon = load_face3d_net(args.face3d_net_path, device)
lm3d_std = load_lm3d('checkpoints/BFM')
video_coeffs = []
for idx in tqdm(range(len(frames_pil)), desc="[Step 2] 3DMM Extraction In Video:"):
frame = frames_pil[idx]
W, H = frame.size
lm_idx = lm[idx].reshape([-1, 2])
if np.mean(lm_idx) == -1:
lm_idx = (lm3d_std[:, :2]+1) / 2.
lm_idx = np.concatenate([lm_idx[:, :1] * W, lm_idx[:, 1:2] * H], 1)
else:
lm_idx[:, -1] = H - 1 - lm_idx[:, -1]
trans_params, im_idx, lm_idx, _ = align_img(frame, lm_idx, lm3d_std)
trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
im_idx_tensor = torch.tensor(np.array(im_idx)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0)
with torch.no_grad():
coeffs = split_coeff(net_recon(im_idx_tensor))
pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs}
pred_coeff = np.concatenate([pred_coeff['id'], pred_coeff['exp'], pred_coeff['tex'], pred_coeff['angle'],\
pred_coeff['gamma'], pred_coeff['trans'], trans_params[None]], 1)
video_coeffs.append(pred_coeff)
semantic_npy = np.array(video_coeffs)[:,0]
np.save('temp/'+base_name+'_coeffs.npy', semantic_npy)
else:
print('[Step 2] Using saved coeffs.')
semantic_npy = np.load('temp/'+base_name+'_coeffs.npy').astype(np.float32)
# generate the 3dmm coeff from a single image
if args.exp_img is not None and ('.png' in args.exp_img or '.jpg' in args.exp_img):
print('extract the exp from',args.exp_img)
exp_pil = Image.open(args.exp_img).convert('RGB')
lm3d_std = load_lm3d('third_part/face3d/BFM')
W, H = exp_pil.size
kp_extractor = KeypointExtractor()
lm_exp = kp_extractor.extract_keypoint([exp_pil], 'temp/'+base_name+'_temp.txt')[0]
if np.mean(lm_exp) == -1:
lm_exp = (lm3d_std[:, :2] + 1) / 2.
lm_exp = np.concatenate(
[lm_exp[:, :1] * W, lm_exp[:, 1:2] * H], 1)
else:
lm_exp[:, -1] = H - 1 - lm_exp[:, -1]
trans_params, im_exp, lm_exp, _ = align_img(exp_pil, lm_exp, lm3d_std)
trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
im_exp_tensor = torch.tensor(np.array(im_exp)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0)
# Ensure face3d net is loaded when deriving expression from an image
if 'net_recon' not in locals():
net_recon = load_face3d_net(args.face3d_net_path, device)
with torch.no_grad():
expression = split_coeff(net_recon(im_exp_tensor))['exp'][0]
del net_recon
elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'neutral':
print('using neutral expression')
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0]
elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'smile':
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_mouth'])[0]
else:
print('using neutral expression (default)')
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0]
# load DNet, model(LNet and ENet)
D_Net, model = load_model(args, device)
if not os.path.isfile('temp/'+base_name+'_stablized.npy') or args.re_preprocess:
imgs = []
for idx in tqdm(range(len(frames_pil)), desc="[Step 3] Stabilize the expression In Video:"):
if args.one_shot:
source_img = trans_image(frames_pil[0]).unsqueeze(0).to(device)
semantic_source_numpy = semantic_npy[0:1]
else:
source_img = trans_image(frames_pil[idx]).unsqueeze(0).to(device)
semantic_source_numpy = semantic_npy[idx:idx+1]
ratio = find_crop_norm_ratio(semantic_source_numpy, semantic_npy)
coeff = transform_semantic(semantic_npy, idx, ratio).unsqueeze(0).to(device)
# hacking the new expression
coeff[:, :64, :] = expression[None, :64, None].to(device)
with torch.no_grad():
output = D_Net(source_img, coeff)
img_stablized = np.uint8((output['fake_image'].squeeze(0).permute(1,2,0).cpu().clamp_(-1, 1).numpy() + 1 )/2. * 255)
imgs.append(cv2.cvtColor(img_stablized,cv2.COLOR_RGB2BGR))
np.save('temp/'+base_name+'_stablized.npy',imgs)
del D_Net
else:
print('[Step 3] Using saved stabilized video.')
imgs = np.load('temp/'+base_name+'_stablized.npy')
torch.cuda.empty_cache()
# Ensure 16kHz mono WAV and clean levels prior to mel extraction
processed_audio_path = 'temp/{}/audio_proc.wav'.format(args.tmp_dir)
if args.audio_preprocess or (not args.audio.endswith('.wav')):
afilters = []
if args.audio_denoise:
afilters.append('afftdn')
if args.audio_normalize:
afilters.append('dynaudnorm=f=150:g=15')
af_clause = ' -af "{}"'.format(','.join(afilters)) if len(afilters) > 0 else ''
command = 'ffmpeg -loglevel error -y -i "{}" -ac 1 -ar {}{} "{}"'.format(
args.audio, args.audio_target_sr, af_clause, processed_audio_path)
subprocess.call(command, shell=True)
args.audio = processed_audio_path
wav = audio.load_wav(args.audio, 16000)
mel = audio.melspectrogram(wav)
if np.isnan(mel.reshape(-1)).sum() > 0:
if args.audio_eps_on_nan:
# Add small epsilon noise (useful for some TTS outputs) and retry once
wav = wav + (1e-6 * np.random.randn(len(wav))).astype(wav.dtype)
mel = audio.melspectrogram(wav)
if np.isnan(mel.reshape(-1)).sum() > 0:
raise ValueError('Mel contains nan! Consider denoising/normalizing or adding epsilon noise and try again')
# Derive mel frames-per-second from audio hyperparams to stay in sync with hop_size
mel_frames_per_second = audio.hp.sample_rate / audio.hp.hop_size
mel_step_size = args.mel_step_size
base_multiplier = (mel_frames_per_second / fps) * args.mel_multiplier_scale
mel_idx_multiplier = base_multiplier
# Build one mel chunk per video frame (1:1 alignment)
def build_mel_chunks(offset_frames, multiplier):
chunks = []
T = mel.shape[1]
for frame_idx in range(len(full_frames)):
start_idx = int(round(frame_idx * multiplier)) + int(offset_frames)
end_idx = start_idx + mel_step_size
if start_idx >= T:
m = np.zeros((mel.shape[0], mel_step_size), dtype=mel.dtype)
elif end_idx > T:
pad = end_idx - T
m = np.pad(mel[:, start_idx:], ((0, 0), (0, pad)), mode='constant')
elif start_idx < 0:
pad = -start_idx
take = mel[:, :max(0, end_idx)]
m = np.pad(take, ((0,0),(pad,0)), mode='constant')
if m.shape[1] < mel_step_size:
m = np.pad(m, ((0,0),(0, mel_step_size - m.shape[1])), mode='constant')
else:
m = mel[:, start_idx:end_idx]
chunks.append(m)
return chunks
base_offset = int(args.mel_offset_frames)
mel_chunks = build_mel_chunks(base_offset, mel_idx_multiplier)
# Optional: auto calibration of global offset by correlating mouth openness with mel energy
if (not args.static) and getattr(args, 'auto_sync_calibrate', False):
try:
# Compute mouth openness per frame from 68-point landmarks
# Using pairs: (62,66), (63,67), (61,65) in 0-based indices
def mouth_open_metric(lms):
if lms is None or len(lms) < 68:
return 0.0
p = lms
pairs = [(62,66), (63,67), (61,65)]
vals = []
for a,b in pairs:
ya, yb = float(p[a][1]), float(p[b][1])
vals.append(abs(yb - ya))
return float(np.mean(vals)) if len(vals) > 0 else 0.0
mouth_series = np.array([mouth_open_metric(land) for land in lm[:len(full_frames)]], dtype=np.float32)
def nz_norm(x):
x = x - x.mean()
s = x.std()
return x / (s if s > 1e-6 else 1.0)
m_norm = nz_norm(mouth_series)
max_lag = int(getattr(args, 'calibrate_window', 6))
def evaluate_alignment(chunks):
energy_series = np.array([float((mc * mc).mean()) for mc in chunks], dtype=np.float32)
e_norm = nz_norm(energy_series)
best_corr_local, best_off_local = -1e9, 0
for off in range(-max_lag, max_lag+1):
if off >= 0:
mn = m_norm[off:]
en = e_norm[:len(m_norm)-off]
else:
mn = m_norm[:off]
en = e_norm[-off:]
L = min(len(mn), len(en))
if L < 10:
continue
c = float(np.corrcoef(mn[:L], en[:L])[0,1])
if c > best_corr_local:
best_corr_local, best_off_local = c, off
return best_corr_local, best_off_local
best_global = -1e9
best_chunks = mel_chunks
best_offset = 0
best_multiplier = mel_idx_multiplier
scale_search = getattr(args, 'auto_sync_scale', False)
if scale_search:
scale_range = float(max(0.0, getattr(args, 'sync_scale_range', 0.08)))
steps = max(1, int(getattr(args, 'sync_scale_steps', 5)))
scales = np.linspace(1.0 - scale_range, 1.0 + scale_range, 2 * steps + 1)
else:
scales = [1.0]
for scale in scales:
candidate_multiplier = base_multiplier * scale
candidate_chunks = build_mel_chunks(base_offset, candidate_multiplier)
corr, off = evaluate_alignment(candidate_chunks)
if corr > best_global:
best_global = corr
best_chunks = candidate_chunks
best_offset = off
best_multiplier = candidate_multiplier
mel_idx_multiplier = best_multiplier
if best_offset != 0:
print(f"[Calibrate] Best offset {best_offset} frames (corr={best_global:.3f}). Rebuilding mel chunks.")
base_offset += best_offset
best_chunks = build_mel_chunks(base_offset, mel_idx_multiplier)
if scale_search and abs(best_multiplier - base_multiplier) > 1e-6:
scale_factor = best_multiplier / base_multiplier if base_multiplier != 0 else 1.0
print(f"[Calibrate] Applied scale factor {scale_factor:.4f} to mel/frame alignment.")
mel_chunks = best_chunks
except Exception as e:
print(f"[Calibrate] Skipped due to error: {e}")
print(f"✅ Built {len(mel_chunks)} mel chunks for {len(full_frames)} frames (multiplier={mel_idx_multiplier:.3f}, offset_frames={base_offset})")
print("[Step 4] Load audio; Length of mel chunks: {}".format(len(mel_chunks)))
imgs_enhanced = []
for idx in tqdm(range(len(imgs)), desc='[Step 5] Reference Enhancement'):
img = imgs[idx]
pred, _, _ = enhancer.process(img, img, face_enhance=True, possion_blending=False)
imgs_enhanced.append(pred)
gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2))
frame_h, frame_w = full_frames[0].shape[:-1]
out = cv2.VideoWriter('temp/{}/result.mp4'.format(args.tmp_dir), cv2.VideoWriter_fourcc(*'mp4v'), fps, (frame_w, frame_h))
if args.up_face != 'original':
instance = GANimationModel()
instance.initialize()
instance.setup()
kp_extractor = KeypointExtractor()
prev_lip_patch = None
prev_face_gray = None
temporal_buffer = []
temporal_window = max(1, int(getattr(args, 'lip_temporal_window', 5)))
lip_mode = str(getattr(args, 'lip_temporal_mode', 'ema')).lower()
# Optional warmup run to stabilize first frames
if getattr(args, 'warmup_frames', 0) and args.warmup_frames > 0:
try:
warm_item = next(gen)
img_batch, mel_batch, frames, coords, img_original, f_frames = warm_item
img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device)
mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device)
img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255.
with torch.no_grad():
incomplete, reference = torch.split(img_batch, 3, dim=1)
_ = model(mel_batch, img_batch, reference)
except StopIteration:
pass
# Recreate generator so we don't lose the first batch
gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2))
for i, (img_batch, mel_batch, frames, coords, img_original, f_frames) in enumerate(tqdm(gen, desc='[Step 6] Lip Synthesis:', total=int(np.ceil(float(len(mel_chunks)) / args.LNet_batch_size)))):
img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device)
mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device)
img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255. # BGR -> RGB
# Validate mel batches to ensure minimum spatial dimensions
valid_indices = []
for bi in range(mel_batch.shape[0]):
sample = mel_batch[bi]
if sample.shape[-1] < 3 or sample.shape[-2] < 3:
print(f"⚠️ Skipping invalid mel chunk shape: {tuple(sample.shape)}")
else:
valid_indices.append(bi)
if len(valid_indices) == 0:
raise ValueError("❌ All mel chunks were invalid. Check audio length or hop_size.")
if len(valid_indices) != mel_batch.shape[0]:
mel_batch = mel_batch[valid_indices]
img_batch = img_batch[valid_indices]
img_original = img_original[valid_indices]
# Also filter associated metadata to keep alignment
frames = [frames[j] for j in valid_indices]
coords = [coords[j] for j in valid_indices]
f_frames = [f_frames[j] for j in valid_indices]
print(f"✅ Proceeding with {mel_batch.shape[0]} valid mel chunks of shape {tuple(mel_batch.shape[-2:])}")
latent_scale = max(0.0, float(getattr(args, 'latent_noise_scale', 0.0)))
if latent_scale > 0.0:
mel_batch = mel_batch + torch.randn_like(mel_batch) * latent_scale
with torch.no_grad():
incomplete, reference = torch.split(img_batch, 3, dim=1)
pred, low_res = model(mel_batch, img_batch, reference)
pred = torch.clamp(pred, 0, 1)
if args.up_face in ['sad', 'angry', 'surprise']:
tar_aus = exp_aus_dict[args.up_face]
else:
pass
if args.up_face == 'original':
cur_gen_faces = img_original
else:
test_batch = {'src_img': torch.nn.functional.interpolate((img_original * 2 - 1), size=(128, 128), mode='bilinear'),
'tar_aus': tar_aus.repeat(len(incomplete), 1)}
instance.feed_batch(test_batch)
instance.forward()
cur_gen_faces = torch.nn.functional.interpolate(
instance.fake_img / 2. + 0.5,
size=(args.img_size, args.img_size),
mode='bilinear'
)
if args.without_rl1 is not False:
incomplete, reference = torch.split(img_batch, 3, dim=1)
mask = torch.where(incomplete==0, torch.ones_like(incomplete), torch.zeros_like(incomplete))
pred = pred * mask + cur_gen_faces * (1 - mask)
pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255.
if args.lip_temporal_smooth:
base_weight = min(max(args.temporal_blend_weight, 0.0), 1.0)
override_smooth = getattr(args, 'mouth_region_smoothing', None)
if override_smooth is not None:
base_weight = float(min(max(override_smooth, 0.0), 1.0))
adaptive = getattr(args, 'temporal_blend_auto', False)
k = float(getattr(args, 'temporal_blend_k', 5.0))
smoothed = []
# Compute per-sample energy from current mel_batch
try:
mel_energy = (mel_batch ** 2).mean(dim=[1,2,3]).detach().cpu().numpy()
e_min, e_max = float(mel_energy.min()), float(mel_energy.max())
denom = (e_max - e_min) if (e_max - e_min) > 1e-6 else 1.0
mel_energy_n = (mel_energy - e_min) / denom
except Exception:
adaptive = False
mel_energy_n = None
# Prepare SyncNet or proxy confidence support
sync_gate = bool(getattr(args, 'enable_syncnet_gate', False))
conf_thr = float(getattr(args, 'lip_sync_conf_threshold', 0.0))
use_gate = sync_gate and conf_thr > 0.0
sync_model = None
if use_gate:
sync_model = _maybe_load_syncnet(getattr(args, 'syncnet_model_path', 'checkpoints/syncnet.pt'))
for idx, p in enumerate(pred):
out_patch = p
# EMA smoothing using previous frame
if lip_mode in ['ema', 'both']:
if prev_lip_patch is not None:
if adaptive and mel_energy_n is not None:
w = base_weight * float(np.exp(-k * mel_energy_n[idx]))
else:
w = base_weight
w = float(min(max(w, 0.0), 1.0))
out_patch = (1.0 - w) * p + w * prev_lip_patch
# Optical-flow guided warping of previous patch to current geometry
if getattr(args, 'flow_temporal_smooth', False) and prev_lip_patch is not None:
try:
# Prepare current face gray image from original face patch
cur_face_tensor = img_original[idx] # shape: 3xH xW (RGB, 0..1)
cur_face = (cur_face_tensor.detach().cpu().numpy().transpose(1, 2, 0) * 255.).astype(np.uint8)
cur_gray = cv2.cvtColor(cur_face, cv2.COLOR_RGB2GRAY)
if prev_face_gray is not None and prev_face_gray.shape == cur_gray.shape:
flow = cv2.calcOpticalFlowFarneback(
prev_face_gray, cur_gray, None,
float(getattr(args, 'flow_pyr_scale', 0.5)),
int(getattr(args, 'flow_levels', 3)),
int(getattr(args, 'flow_winsize', 21)),
int(getattr(args, 'flow_iters', 3)),
int(getattr(args, 'flow_poly_n', 5)),
float(getattr(args, 'flow_poly_sigma', 1.2)),
0
)
h, w = cur_gray.shape
grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))
map_x = (grid_x + flow[..., 0]).astype(np.float32)
map_y = (grid_y + flow[..., 1]).astype(np.float32)
warped_prev = cv2.remap(prev_lip_patch.astype(np.float32), map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
wf = float(getattr(args, 'flow_blend_weight', 0.4))
if override_smooth is not None:
wf = float(min(max(override_smooth, 0.0), 1.0))
wf = max(0.0, min(1.0, wf))
out_patch = (1.0 - wf) * out_patch + wf * warped_prev
# update prev_face_gray for next iteration
prev_face_gray = cur_gray
except Exception:
# if any failure in flow, just skip silently
pass
# Median over a small temporal window to remove flicker
if lip_mode in ['median', 'both']:
temporal_buffer.append(out_patch)
if len(temporal_buffer) > temporal_window:
temporal_buffer.pop(0)
stack = np.stack(temporal_buffer, axis=0)
out_patch = np.median(stack, axis=0)
# Sync confidence gating: hold more of previous patch if confidence is low
if use_gate and prev_lip_patch is not None:
try:
if sync_model is not None:
# Placeholder: if a TorchScript syncnet returns a scalar conf for current sample
# Users can swap this with their actual model interface
# Example expects input shapes to match model requirements; adjust as needed
with torch.no_grad():
# Provide minimal placeholders; real implementation should pass mouth ROI and mel window
conf_tensor = torch.tensor([float(mel_energy_n[idx])], dtype=torch.float32)
conf = float(conf_tensor.item())
else:
# Proxy confidence: normalized mel energy for this frame
conf = float(mel_energy_n[idx]) if mel_energy_n is not None else 1.0
if conf < conf_thr:
hold_w = min(0.95, max(0.5, 0.5 + (conf_thr - conf)))
out_patch = hold_w * prev_lip_patch + (1.0 - hold_w) * out_patch
except Exception:
pass
smoothed.append(out_patch)
prev_lip_patch = out_patch
pred = np.stack(smoothed, axis=0)
torch.cuda.empty_cache()
for p, f, xf, c in zip(pred, frames, f_frames, coords):
y1, y2, x1, x2 = c
p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1))
ff = xf.copy()
ff[y1:y2, x1:x2] = p
# month region enhancement by GFPGAN
cropped_faces, restored_faces, restored_img = restorer.enhance(
ff, has_aligned=False, only_center_face=True, paste_back=True)
# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
mm = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0]
mouse_mask = np.zeros_like(restored_img)
tmp_mask = enhancer.faceparser.process(restored_img[y1:y2, x1:x2], mm)[0]
tmp_mask = mask_postprocess(tmp_mask, thres=getattr(args, 'mask_feather', 20))
mouse_mask[y1:y2, x1:x2]= cv2.resize(tmp_mask, (x2 - x1, y2 - y1))[:, :, np.newaxis] / 255.
height, width = ff.shape[:2]
# Ensure shapes align for blending at native resolution
restored_img = restored_img if restored_img.shape[:2] == (height, width) else cv2.resize(restored_img, (width, height))
ff = ff if ff.shape[:2] == (height, width) else cv2.resize(ff, (width, height))
full_mask = np.float32(mouse_mask)
if full_mask.shape[:2] != (height, width):
full_mask = cv2.resize(full_mask, (width, height))
# --- Shape alignment fix for Laplacian blending ---
if restored_img.shape != ff.shape:
ff = cv2.resize(ff, (restored_img.shape[1], restored_img.shape[0]))
print(f"[Fix] Resized ff to {ff.shape} to match restored_img")
if full_mask.shape[:2] != restored_img.shape[:2]:
full_mask = cv2.resize(full_mask, (restored_img.shape[1], restored_img.shape[0]))
print(f"[Fix] Resized full_mask to {full_mask.shape} to match restored_img")
print("Shapes before blending:", restored_img.shape, ff.shape, full_mask.shape)
img = Laplacian_Pyramid_Blending_with_mask(restored_img, ff, full_mask[:, :, 0], 12)
img = np.clip(img, 0, 255)
if img.shape[:2] != (height, width):
pp = np.uint8(cv2.resize(img, (width, height)))
else:
pp = np.uint8(img)
pp, orig_faces, enhanced_faces = enhancer.process(pp, xf, bbox=c, face_enhance=True, possion_blending=False)
out.write(pp)
out.release()
if not os.path.isdir(os.path.dirname(args.outfile)):
os.makedirs(os.path.dirname(args.outfile), exist_ok=True)
if platform.system() == 'Windows':
command = ['ffmpeg', '-loglevel', 'error', '-y', '-i', args.audio, '-i', 'temp/{}/result.mp4'.format(args.tmp_dir), '-shortest', '-c:v', 'copy', args.outfile]
subprocess.call(command, shell=False)
else:
command = 'ffmpeg -loglevel error -y -i {} -i {} -shortest -c:v copy {}'.format(args.audio, 'temp/{}/result.mp4'.format(args.tmp_dir), args.outfile)
subprocess.call(command, shell=True)
print('outfile:', args.outfile)
# frames:256x256, full_frames: original size
def datagen(frames, mels, full_frames, frames_pil, cox):
img_batch, mel_batch, frame_batch, coords_batch, ref_batch, full_frame_batch = [], [], [], [], [], []
base_name = args.face.split('/')[-1]
refs = []
image_size = 256
# original frames
kp_extractor = KeypointExtractor()
fr_pil = [Image.fromarray(frame) for frame in frames]
lms = kp_extractor.extract_keypoint(fr_pil, 'temp/'+base_name+'x12_landmarks.txt')
frames_pil = [ (lm, frame) for frame,lm in zip(fr_pil, lms)] # frames is the croped version of modified face
crops, orig_images, quads = crop_faces(image_size, frames_pil, scale=1.0, use_fa=True)
# Guard against invalid (NaN/inf) quads by reusing last valid or falling back to identity
clean_quads = []
identity_quad = np.array([[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]], dtype=np.float32)
for q in quads:
if q is None or (not np.isfinite(q).all()):
q = clean_quads[-1] if len(clean_quads) > 0 else identity_quad
clean_quads.append(q)
# EMA smooth the alignment quads to reduce temporal jitter in warping
beta_q = float(getattr(args, 'quad_ema_beta', 0.0))
if beta_q > 0.0 and len(clean_quads) > 1:
sm_quads = []
prev_q = clean_quads[0].astype(np.float32)
sm_quads.append(prev_q)
for q in clean_quads[1:]:
qf = q.astype(np.float32)
prev_q = beta_q * prev_q + (1.0 - beta_q) * qf
sm_quads.append(prev_q)
else:
sm_quads = clean_quads
inverse_transforms = [calc_alignment_coefficients(q + 0.5, [[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]]) for q in sm_quads]
del kp_extractor.detector
oy1,oy2,ox1,ox2 = cox
face_det_results = face_detect(full_frames, args, jaw_correction=True)
for inverse_transform, crop, full_frame, face_det in zip(inverse_transforms, crops, full_frames, face_det_results):
imc_pil = paste_image(inverse_transform, crop, Image.fromarray(
cv2.resize(full_frame[int(oy1):int(oy2), int(ox1):int(ox2)], (256, 256))))
ff = full_frame.copy()
ff[int(oy1):int(oy2), int(ox1):int(ox2)] = cv2.resize(np.array(imc_pil.convert('RGB')), (ox2 - ox1, oy2 - oy1))
oface, coords = face_det
y1, y2, x1, x2 = coords
refs.append(ff[y1: y2, x1:x2])
for i, m in enumerate(mels):
idx = 0 if args.static else i
frame_to_save = frames[idx].copy()
face = refs[idx]
oface, coords = face_det_results[idx].copy()
face = cv2.resize(face, (args.img_size, args.img_size))
oface = cv2.resize(oface, (args.img_size, args.img_size))
img_batch.append(oface)
ref_batch.append(face)
mel_batch.append(m)
coords_batch.append(coords)
frame_batch.append(frame_to_save)
full_frame_batch.append(full_frames[idx].copy())
if len(img_batch) >= args.LNet_batch_size:
img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch)
img_masked = img_batch.copy()
img_original = img_batch.copy()
img_masked[:, args.img_size//2:] = 0
img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255.
mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])
yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch
img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch, ref_batch = [], [], [], [], [], [], []
if len(img_batch) > 0:
img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch)
img_masked = img_batch.copy()
img_original = img_batch.copy()
img_masked[:, args.img_size//2:] = 0
img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255.
mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])
yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch
if __name__ == '__main__':
main()
|