Download inference.py from Anirudh1950/VideoReTalk: direct link, hf CLI and curl.
- Browser
- Download file 52.4 kB
-
https://huggingface.co/Anirudh1950/VideoReTalk/resolve/main/inference.py
- Command line
-
hf download hf://Anirudh1950/VideoReTalk/inference.py
-
curl -L -o inference.py https://huggingface.co/Anirudh1950/VideoReTalk/resolve/main/inference.py
52.4 kB
| import numpy as np | |
| import cv2, os, sys, subprocess, platform, torch | |
| import math | |
| from tqdm import tqdm | |
| from PIL import Image | |
| from scipy.io import loadmat | |
| sys.path.insert(0, 'third_part') | |
| sys.path.insert(0, 'third_part/GPEN') | |
| sys.path.insert(0, 'third_part/GFPGAN') | |
| # 3dmm extraction | |
| from third_part.face3d.util.preprocess import align_img | |
| from third_part.face3d.util.load_mats import load_lm3d | |
| from third_part.face3d.extract_kp_videos import KeypointExtractor | |
| # face enhancement | |
| from third_part.GPEN.gpen_face_enhancer import FaceEnhancement | |
| from third_part.GFPGAN.gfpgan import GFPGANer | |
| # expression control | |
| from third_part.ganimation_replicate.model.ganimation import GANimationModel | |
| from utils import audio | |
| from utils.ffhq_preprocess import Croper | |
| from utils.alignment_stit import crop_faces, calc_alignment_coefficients, paste_image | |
| from utils.inference_utils import Laplacian_Pyramid_Blending_with_mask, face_detect, load_model, options, split_coeff, \ | |
| trans_image, transform_semantic, find_crop_norm_ratio, load_face3d_net, exp_aus_dict, mask_postprocess | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| args = options() | |
| # Optional: attempt to load a SyncNet model if provided; otherwise fall back to proxy confidence | |
| _syncnet = None | |
| _syncnet_warned = False | |
| def _maybe_load_syncnet(path): | |
| global _syncnet, _syncnet_warned | |
| if _syncnet is not None: | |
| return _syncnet | |
| try: | |
| if os.path.isfile(path): | |
| # Expect a TorchScript or torch.load-able module that exposes forward(audio, video)->embeds or confidence | |
| _syncnet = torch.jit.load(path, map_location='cpu').eval() | |
| else: | |
| if not _syncnet_warned: | |
| print(f"[SyncNet] Model not found at {path}. Using proxy confidence.") | |
| _syncnet_warned = True | |
| _syncnet = None | |
| except Exception as e: | |
| if not _syncnet_warned: | |
| print(f"[SyncNet] Failed to load model ({e}). Using proxy confidence.") | |
| _syncnet_warned = True | |
| _syncnet = None | |
| return _syncnet | |
| def _ffprobe_duration(path): | |
| try: | |
| cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nk=1:nw=1', path] | |
| out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) | |
| return float(out.decode().strip()) | |
| except Exception as e: | |
| print(f"[Warn] ffprobe failed for {path}: {e}") | |
| return None | |
| def _split_into_chunks(face_path, audio_path, out_dir, fps, chunk_duration, overlap, enforce_sr=16000, crossfade=False): | |
| os.makedirs(out_dir, exist_ok=True) | |
| video_dur = _ffprobe_duration(face_path) or 0 | |
| audio_dur = _ffprobe_duration(audio_path) or 0 | |
| total_dur = min(video_dur, audio_dur) if (video_dur > 0 and audio_dur > 0) else max(video_dur, audio_dur) | |
| if total_dur == 0: | |
| raise RuntimeError('Could not determine media duration') | |
| eff_overlap = overlap if crossfade else 0.0 | |
| step = max(0.0, chunk_duration - eff_overlap) if chunk_duration > 0 else total_dur | |
| starts = [] | |
| t = 0.0 | |
| while t < total_dur: | |
| starts.append(t) | |
| t += step if step > 0 else total_dur | |
| chunk_paths = [] | |
| for idx, start in enumerate(starts): | |
| dur = min(chunk_duration if chunk_duration > 0 else total_dur, total_dur - start) | |
| v_out = os.path.join(out_dir, f'video_{idx+1:03d}.mp4') | |
| a_out = os.path.join(out_dir, f'audio_{idx+1:03d}.wav') | |
| # Re-encode video chunk to enforce exact boundaries & fps | |
| vcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', face_path, '-t', f'{dur:.3f}', | |
| '-r', f'{fps:.3f}', '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', '-pix_fmt', 'yuv420p', v_out] | |
| acmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', audio_path, '-t', f'{dur:.3f}', | |
| '-ac', '1', '-ar', str(enforce_sr), a_out] | |
| try: | |
| subprocess.check_call(vcmd, shell=False) | |
| subprocess.check_call(acmd, shell=False) | |
| except subprocess.CalledProcessError as e: | |
| print(f"[Warn] Failed to split chunk {idx+1}: {e}") | |
| continue | |
| # Validate audio sample rate | |
| try: | |
| sr = _probe_audio_sr(a_out) | |
| if sr != enforce_sr: | |
| print(f"[Warn] Fixing sample rate for {a_out} (got {sr}); forcing {enforce_sr}") | |
| fixcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', a_out, '-ac', '1', '-ar', str(enforce_sr), a_out] | |
| subprocess.check_call(fixcmd, shell=False) | |
| except Exception as e: | |
| print(f"[Warn] Could not validate audio sr for {a_out}: {e}") | |
| chunk_paths.append((v_out, a_out)) | |
| return chunk_paths | |
| def _probe_audio_sr(path): | |
| cmd = ['ffprobe', '-v', 'error', '-select_streams', 'a:0', '-show_entries', 'stream=sample_rate', '-of', 'default=nk=1:nw=1', path] | |
| out = subprocess.check_output(cmd, stderr=subprocess.STDOUT) | |
| return int(out.decode().strip()) | |
| def _concat_media(list_file, output_path, stream='video', reencode=False, drop_audio=False): | |
| # stream: 'video' or 'audio' | |
| if stream == 'video': | |
| if reencode: | |
| cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c:v', 'libx264', '-pix_fmt', 'yuv420p'] | |
| else: | |
| cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy'] | |
| if drop_audio: | |
| cmd += ['-an'] | |
| cmd += [output_path] | |
| else: | |
| cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy', output_path] | |
| subprocess.check_call(cmd, shell=False) | |
| def _xfade_concatenate(videos, out_path, overlap): | |
| # Simple pairwise xfade across N segments | |
| if len(videos) == 1: | |
| return videos[0] | |
| cur = videos[0] | |
| for i in range(1, len(videos)): | |
| nxt = videos[i] | |
| tmp = out_path.replace('.mp4', f'_xf{i:03d}.mp4') | |
| cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', cur, '-i', nxt, | |
| '-filter_complex', f"[0:v][1:v]xfade=transition=fade:duration={overlap:.3f}:offset=PTS-STARTPTS[v];[0:a][1:a]acrossfade=d={overlap:.3f}[a]", | |
| '-map', '[v]', '-map', '[a]', tmp] | |
| subprocess.check_call(cmd, shell=False) | |
| cur = tmp | |
| os.replace(cur, out_path) | |
| return out_path | |
| # --- Video stabilization helpers --- | |
| def _stabilize_video_with_vidstab(input_path, out_dir, shakiness=5, smoothing=30, crop_mode='keep'): | |
| os.makedirs(out_dir, exist_ok=True) | |
| trf_path = os.path.join(out_dir, 'vidstab_transform.trf') | |
| stabilized_path = os.path.join(out_dir, 'stabilized_input.mp4') | |
| # Normalize paths for ffmpeg filters | |
| def _norm(p): | |
| return os.path.abspath(p).replace('\\', '/') | |
| input_norm = _norm(input_path) | |
| trf_norm = _norm(trf_path) | |
| # Pass 1: detect | |
| detect_filter = f"vidstabdetect=shakiness={int(shakiness)}:accuracy=15:result={trf_norm}" | |
| detect_cmd = ['ffmpeg', '-loglevel', 'error', '-y', | |
| '-i', input_norm, | |
| '-vf', detect_filter, | |
| '-f', 'null', '-'] | |
| # Pass 2: transform | |
| crop_mode = crop_mode if crop_mode in ['keep', 'content', 'black'] else 'keep' | |
| transform_filter = f"vidstabtransform=smoothing={int(smoothing)}:input={trf_norm}:crop={crop_mode}" | |
| transform_cmd = ['ffmpeg', '-loglevel', 'error', '-y', | |
| '-i', input_norm, | |
| '-vf', transform_filter, | |
| '-c:v', 'libx264', '-preset', 'medium', '-crf', '18', | |
| '-c:a', 'copy', | |
| _norm(stabilized_path)] | |
| try: | |
| subprocess.check_call(detect_cmd, shell=False) | |
| subprocess.check_call(transform_cmd, shell=False) | |
| except subprocess.CalledProcessError as e: | |
| raise RuntimeError(f'ffmpeg vidstab failed: {e}') | |
| return stabilized_path | |
| # --- Sync gating helpers --- | |
| def _mouth_open_metric(lms): | |
| try: | |
| if lms is None or len(lms) < 68: | |
| return 0.0 | |
| p = lms | |
| pairs = [(62,66), (63,67), (61,65)] | |
| vals = [] | |
| for a, b in pairs: | |
| ya, yb = float(p[a][1]), float(p[b][1]) | |
| vals.append(abs(yb - ya)) | |
| return float(np.mean(vals)) if len(vals) > 0 else 0.0 | |
| except Exception: | |
| return 0.0 | |
| def _sync_confidence(frame_idx, lm_all, mel_chunks, win=5): | |
| try: | |
| n = len(mel_chunks) | |
| if n == 0: | |
| return 0.0 | |
| s = max(0, frame_idx - win//2) | |
| e = min(n, frame_idx + (win - win//2)) | |
| if e - s < 3: | |
| return 0.0 | |
| mouth_series = np.array([_mouth_open_metric(lm_all[k]) for k in range(s, e)], dtype=np.float32) | |
| energy_series = np.array([float((mel_chunks[k].astype(np.float32) ** 2).mean()) for k in range(s, e)], dtype=np.float32) | |
| # normalize | |
| def nz_norm(x): | |
| x = x - x.mean() | |
| sdev = x.std() | |
| return x / (sdev if sdev > 1e-6 else 1.0) | |
| m_norm = nz_norm(mouth_series) | |
| e_norm = nz_norm(energy_series) | |
| L = min(len(m_norm), len(e_norm)) | |
| if L < 3: | |
| return 0.0 | |
| corr = float(np.corrcoef(m_norm[:L], e_norm[:L])[0, 1]) | |
| return max(0.0, min(1.0, 0.5 * (corr + 1.0))) | |
| except Exception: | |
| return 0.0 | |
| def main(): | |
| device = 'cuda' if torch.cuda.is_available() else 'cpu' | |
| print('[Info] Using {} for inference.'.format(device)) | |
| os.makedirs(os.path.join('temp', args.tmp_dir), exist_ok=True) | |
| enhancer = FaceEnhancement(base_dir='checkpoints', size=args.gpen_size, model=args.gpen_model, use_sr=False, \ | |
| sr_model='rrdb_realesrnet_psnr', channel_multiplier=2, narrow=1, device=device) | |
| restorer = GFPGANer(model_path='checkpoints/GFPGANv1.3.pth', upscale=1, arch='clean', \ | |
| channel_multiplier=2, bg_upsampler=None) | |
| base_name = args.face.split('/')[-1] | |
| if os.path.isfile(args.face) and args.face.split('.')[1] in ['jpg', 'png', 'jpeg']: | |
| args.static = True | |
| # Enforce one-shot for static images | |
| if not args.one_shot: | |
| print('[Info] Static image detected; enabling one-shot mode.') | |
| args.one_shot = True | |
| if not os.path.isfile(args.face): | |
| raise ValueError('--face argument must be a valid path to video/image file') | |
| elif args.face.split('.')[1] in ['jpg', 'png', 'jpeg']: | |
| full_frames = [cv2.imread(args.face)] | |
| fps = args.fps | |
| else: | |
| # Disable one-shot for videos | |
| if args.one_shot: | |
| print('[Info] Video detected; disabling one-shot mode (only for static images).') | |
| args.one_shot = False | |
| if getattr(args, 'pre_stabilize_video', False): | |
| try: | |
| stabilize_dir = os.path.join('temp', args.tmp_dir, 'stabilize') | |
| stabilized_face = _stabilize_video_with_vidstab( | |
| args.face, | |
| stabilize_dir, | |
| shakiness=getattr(args, 'stabilize_shakiness', 5), | |
| smoothing=getattr(args, 'stabilize_smoothing', 30), | |
| crop_mode=getattr(args, 'stabilize_crop', 'keep') | |
| ) | |
| print(f"[Stabilize] Using stabilized video at {stabilized_face}") | |
| args.face = stabilized_face | |
| except Exception as e: | |
| print(f"[Stabilize] Warning: stabilization failed ({e}). Proceeding with original video.") | |
| # Chunked processing for long videos | |
| if args.enable_chunking and args.chunk_duration and args.chunk_duration > 0: | |
| try: | |
| video_dur = _ffprobe_duration(args.face) or 0 | |
| if video_dur > args.chunk_duration + 0.5: | |
| # Probe fps | |
| cap_tmp = cv2.VideoCapture(args.face) | |
| fps_probe = cap_tmp.get(cv2.CAP_PROP_FPS) | |
| cap_tmp.release() | |
| if not fps_probe or math.isclose(fps_probe, 0.0) or np.isnan(fps_probe): | |
| fps_probe = args.fps | |
| parts_dir = os.path.join('temp', args.tmp_dir, args.chunk_temp_dir) | |
| out_parts_dir = os.path.join('temp', args.tmp_dir, 'out_parts') | |
| os.makedirs(parts_dir, exist_ok=True) | |
| os.makedirs(out_parts_dir, exist_ok=True) | |
| print(f"[Chunk] Splitting into ~{args.chunk_duration}s chunks with overlap={args.chunk_overlap}s") | |
| chunks = _split_into_chunks(args.face, args.audio, parts_dir, fps_probe, args.chunk_duration, args.chunk_overlap, enforce_sr=16000, crossfade=args.chunk_crossfade) | |
| # Minimal required seconds for a mel window | |
| from utils import audio as _audio | |
| min_sec = (args.mel_step_size * _audio.hp.hop_size) / _audio.hp.sample_rate | |
| successes = [] | |
| for ci, (vpath, apath) in enumerate(chunks, start=1): | |
| try: | |
| # Validate length | |
| c_dur = _ffprobe_duration(vpath) or 0 | |
| if c_dur < min_sec: | |
| print(f"[Warn] Skipping chunk {ci:03d} too short: {c_dur:.3f}s < {min_sec:.3f}s") | |
| continue | |
| out_path = os.path.join(out_parts_dir, f'output_{ci:03d}.mp4') | |
| cmd = [sys.executable, os.path.abspath(__file__), '--face', vpath, '--audio', apath, '--outfile', out_path, '--fps', f'{fps_probe:.3f}', '--tmp_dir', f"{args.tmp_dir}/chunk_{ci:03d}", '--chunk_duration', '0'] | |
| # Propagate useful flags | |
| if args.audio_preprocess: cmd.append('--audio_preprocess') | |
| if args.audio_normalize: cmd.append('--audio_normalize') | |
| if args.audio_denoise: cmd.append('--audio_denoise') | |
| if args.lip_temporal_smooth: cmd.append('--lip_temporal_smooth') | |
| # Always re-preprocess per chunk to avoid stale landmark/cached files mismatch | |
| cmd.append('--re_preprocess') | |
| try: | |
| subprocess.check_call(cmd, shell=False) | |
| successes.append(out_path) | |
| except subprocess.CalledProcessError as e: | |
| print(f"[Warn] Chunk {ci:03d} failed: {e}") | |
| continue | |
| except Exception as e: | |
| print(f"[Warn] Exception on chunk {ci:03d}: {e}") | |
| continue | |
| if len(successes) == 0: | |
| raise RuntimeError('No chunk processed successfully') | |
| final_out = args.outfile | |
| if args.chunk_crossfade: | |
| # Build crossfaded concatenation including audio | |
| concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4') | |
| _xfade_concatenate(successes, concat_video, max(0.0, args.chunk_overlap)) | |
| # Just move to final (already has audio) | |
| if not os.path.isdir(os.path.dirname(final_out)): | |
| os.makedirs(os.path.dirname(final_out), exist_ok=True) | |
| if platform.system() == 'Windows': | |
| subprocess.check_call(['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-c', 'copy', final_out], shell=False) | |
| else: | |
| subprocess.check_call(f'ffmpeg -loglevel error -y -i "{concat_video}" -c copy "{final_out}"', shell=True) | |
| else: | |
| # Concatenate videos (drop audio) | |
| concat_v_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_v.txt') | |
| with open(concat_v_list, 'w', encoding='utf-8') as f: | |
| for p in successes: | |
| _safe = os.path.abspath(p).replace('\\', '/') | |
| f.write(f"file '{_safe}'\n") | |
| concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4') | |
| _concat_media(concat_v_list, concat_video, stream='video', reencode=False, drop_audio=True) | |
| # Concatenate audios | |
| chunk_audios = [os.path.join(parts_dir, f) for f in sorted(os.listdir(parts_dir)) if f.lower().startswith('audio_') and f.lower().endswith('.wav')] | |
| concat_a_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_a.txt') | |
| with open(concat_a_list, 'w', encoding='utf-8') as f: | |
| for p in chunk_audios: | |
| _safe = os.path.abspath(p).replace('\\', '/') | |
| f.write(f"file '{_safe}'\n") | |
| concat_audio = os.path.join('temp', args.tmp_dir, 'concat_audio.wav') | |
| _concat_media(concat_a_list, concat_audio, stream='audio', reencode=False) | |
| # Merge A/V with -shortest | |
| if not os.path.isdir(os.path.dirname(final_out)): | |
| os.makedirs(os.path.dirname(final_out), exist_ok=True) | |
| if platform.system() == 'Windows': | |
| mcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-i', concat_audio, '-shortest', '-c:v', 'copy', final_out] | |
| subprocess.call(mcmd, shell=False) | |
| else: | |
| mcmd = f'ffmpeg -loglevel error -y -i "{concat_video}" -i "{concat_audio}" -shortest -c:v copy "{final_out}"' | |
| subprocess.call(mcmd, shell=True) | |
| print('outfile:', final_out) | |
| return | |
| except Exception as e: | |
| print(f"[Warn] Chunked processing fallback due to error: {e}") | |
| video_stream = cv2.VideoCapture(args.face) | |
| fps = video_stream.get(cv2.CAP_PROP_FPS) | |
| full_frames = [] | |
| while True: | |
| still_reading, frame = video_stream.read() | |
| if not still_reading: | |
| video_stream.release() | |
| break | |
| y1, y2, x1, x2 = args.crop | |
| if x2 == -1: x2 = frame.shape[1] | |
| if y2 == -1: y2 = frame.shape[0] | |
| frame = frame[y1:y2, x1:x2] | |
| full_frames.append(frame) | |
| print ("[Step 0] Number of frames available for inference: "+str(len(full_frames))) | |
| # face detection & cropping, cropping the first frame as the style of FFHQ | |
| croper = Croper('checkpoints/shape_predictor_68_face_landmarks.dat') | |
| full_frames_RGB = [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames] | |
| full_frames_RGB, crop, quad = croper.crop(full_frames_RGB, xsize=512) | |
| clx, cly, crx, cry = crop | |
| lx, ly, rx, ry = quad | |
| lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry) | |
| oy1, oy2, ox1, ox2 = cly+ly, min(cly+ry, full_frames[0].shape[0]), clx+lx, min(clx+rx, full_frames[0].shape[1]) | |
| # original_size = (ox2 - ox1, oy2 - oy1) | |
| frames_pil = [Image.fromarray(cv2.resize(frame,(256,256))) for frame in full_frames_RGB] | |
| # get the landmark according to the detected face. | |
| if not os.path.isfile('temp/'+base_name+'_landmarks.txt') or args.re_preprocess: | |
| print('[Step 1] Landmarks Extraction in Video.') | |
| kp_extractor = KeypointExtractor() | |
| lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt') | |
| else: | |
| print('[Step 1] Using saved landmarks.') | |
| lm_path = 'temp/'+base_name+'_landmarks.txt' | |
| try: | |
| lm_raw = np.loadtxt(lm_path).astype(np.float32) | |
| except Exception as e: | |
| print(f"[Step 1] Failed to load saved landmarks ({e}); re-extracting.") | |
| kp_extractor = KeypointExtractor() | |
| lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt') | |
| else: | |
| total_vals = lm_raw.size | |
| denom = 2 * len(full_frames) | |
| if denom > 0 and total_vals % denom == 0: | |
| k = total_vals // denom | |
| try: | |
| lm = lm_raw.reshape([len(full_frames), k, 2]) | |
| except Exception: | |
| print('[Step 1] Saved landmarks shape mismatch; re-extracting.') | |
| kp_extractor = KeypointExtractor() | |
| lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt') | |
| else: | |
| print('[Step 1] Saved landmarks length does not match frames; re-extracting.') | |
| kp_extractor = KeypointExtractor() | |
| lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt') | |
| if not os.path.isfile('temp/'+base_name+'_coeffs.npy') or args.exp_img is not None or args.re_preprocess: | |
| net_recon = load_face3d_net(args.face3d_net_path, device) | |
| lm3d_std = load_lm3d('checkpoints/BFM') | |
| video_coeffs = [] | |
| for idx in tqdm(range(len(frames_pil)), desc="[Step 2] 3DMM Extraction In Video:"): | |
| frame = frames_pil[idx] | |
| W, H = frame.size | |
| lm_idx = lm[idx].reshape([-1, 2]) | |
| if np.mean(lm_idx) == -1: | |
| lm_idx = (lm3d_std[:, :2]+1) / 2. | |
| lm_idx = np.concatenate([lm_idx[:, :1] * W, lm_idx[:, 1:2] * H], 1) | |
| else: | |
| lm_idx[:, -1] = H - 1 - lm_idx[:, -1] | |
| trans_params, im_idx, lm_idx, _ = align_img(frame, lm_idx, lm3d_std) | |
| trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32) | |
| im_idx_tensor = torch.tensor(np.array(im_idx)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0) | |
| with torch.no_grad(): | |
| coeffs = split_coeff(net_recon(im_idx_tensor)) | |
| pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs} | |
| pred_coeff = np.concatenate([pred_coeff['id'], pred_coeff['exp'], pred_coeff['tex'], pred_coeff['angle'],\ | |
| pred_coeff['gamma'], pred_coeff['trans'], trans_params[None]], 1) | |
| video_coeffs.append(pred_coeff) | |
| semantic_npy = np.array(video_coeffs)[:,0] | |
| np.save('temp/'+base_name+'_coeffs.npy', semantic_npy) | |
| else: | |
| print('[Step 2] Using saved coeffs.') | |
| semantic_npy = np.load('temp/'+base_name+'_coeffs.npy').astype(np.float32) | |
| # generate the 3dmm coeff from a single image | |
| if args.exp_img is not None and ('.png' in args.exp_img or '.jpg' in args.exp_img): | |
| print('extract the exp from',args.exp_img) | |
| exp_pil = Image.open(args.exp_img).convert('RGB') | |
| lm3d_std = load_lm3d('third_part/face3d/BFM') | |
| W, H = exp_pil.size | |
| kp_extractor = KeypointExtractor() | |
| lm_exp = kp_extractor.extract_keypoint([exp_pil], 'temp/'+base_name+'_temp.txt')[0] | |
| if np.mean(lm_exp) == -1: | |
| lm_exp = (lm3d_std[:, :2] + 1) / 2. | |
| lm_exp = np.concatenate( | |
| [lm_exp[:, :1] * W, lm_exp[:, 1:2] * H], 1) | |
| else: | |
| lm_exp[:, -1] = H - 1 - lm_exp[:, -1] | |
| trans_params, im_exp, lm_exp, _ = align_img(exp_pil, lm_exp, lm3d_std) | |
| trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32) | |
| im_exp_tensor = torch.tensor(np.array(im_exp)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0) | |
| # Ensure face3d net is loaded when deriving expression from an image | |
| if 'net_recon' not in locals(): | |
| net_recon = load_face3d_net(args.face3d_net_path, device) | |
| with torch.no_grad(): | |
| expression = split_coeff(net_recon(im_exp_tensor))['exp'][0] | |
| del net_recon | |
| elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'neutral': | |
| print('using neutral expression') | |
| expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0] | |
| elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'smile': | |
| expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_mouth'])[0] | |
| else: | |
| print('using neutral expression (default)') | |
| expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0] | |
| # load DNet, model(LNet and ENet) | |
| D_Net, model = load_model(args, device) | |
| if not os.path.isfile('temp/'+base_name+'_stablized.npy') or args.re_preprocess: | |
| imgs = [] | |
| for idx in tqdm(range(len(frames_pil)), desc="[Step 3] Stabilize the expression In Video:"): | |
| if args.one_shot: | |
| source_img = trans_image(frames_pil[0]).unsqueeze(0).to(device) | |
| semantic_source_numpy = semantic_npy[0:1] | |
| else: | |
| source_img = trans_image(frames_pil[idx]).unsqueeze(0).to(device) | |
| semantic_source_numpy = semantic_npy[idx:idx+1] | |
| ratio = find_crop_norm_ratio(semantic_source_numpy, semantic_npy) | |
| coeff = transform_semantic(semantic_npy, idx, ratio).unsqueeze(0).to(device) | |
| # hacking the new expression | |
| coeff[:, :64, :] = expression[None, :64, None].to(device) | |
| with torch.no_grad(): | |
| output = D_Net(source_img, coeff) | |
| img_stablized = np.uint8((output['fake_image'].squeeze(0).permute(1,2,0).cpu().clamp_(-1, 1).numpy() + 1 )/2. * 255) | |
| imgs.append(cv2.cvtColor(img_stablized,cv2.COLOR_RGB2BGR)) | |
| np.save('temp/'+base_name+'_stablized.npy',imgs) | |
| del D_Net | |
| else: | |
| print('[Step 3] Using saved stabilized video.') | |
| imgs = np.load('temp/'+base_name+'_stablized.npy') | |
| torch.cuda.empty_cache() | |
| # Ensure 16kHz mono WAV and clean levels prior to mel extraction | |
| processed_audio_path = 'temp/{}/audio_proc.wav'.format(args.tmp_dir) | |
| if args.audio_preprocess or (not args.audio.endswith('.wav')): | |
| afilters = [] | |
| if args.audio_denoise: | |
| afilters.append('afftdn') | |
| if args.audio_normalize: | |
| afilters.append('dynaudnorm=f=150:g=15') | |
| af_clause = ' -af "{}"'.format(','.join(afilters)) if len(afilters) > 0 else '' | |
| command = 'ffmpeg -loglevel error -y -i "{}" -ac 1 -ar {}{} "{}"'.format( | |
| args.audio, args.audio_target_sr, af_clause, processed_audio_path) | |
| subprocess.call(command, shell=True) | |
| args.audio = processed_audio_path | |
| wav = audio.load_wav(args.audio, 16000) | |
| mel = audio.melspectrogram(wav) | |
| if np.isnan(mel.reshape(-1)).sum() > 0: | |
| if args.audio_eps_on_nan: | |
| # Add small epsilon noise (useful for some TTS outputs) and retry once | |
| wav = wav + (1e-6 * np.random.randn(len(wav))).astype(wav.dtype) | |
| mel = audio.melspectrogram(wav) | |
| if np.isnan(mel.reshape(-1)).sum() > 0: | |
| raise ValueError('Mel contains nan! Consider denoising/normalizing or adding epsilon noise and try again') | |
| # Derive mel frames-per-second from audio hyperparams to stay in sync with hop_size | |
| mel_frames_per_second = audio.hp.sample_rate / audio.hp.hop_size | |
| mel_step_size = args.mel_step_size | |
| base_multiplier = (mel_frames_per_second / fps) * args.mel_multiplier_scale | |
| mel_idx_multiplier = base_multiplier | |
| # Build one mel chunk per video frame (1:1 alignment) | |
| def build_mel_chunks(offset_frames, multiplier): | |
| chunks = [] | |
| T = mel.shape[1] | |
| for frame_idx in range(len(full_frames)): | |
| start_idx = int(round(frame_idx * multiplier)) + int(offset_frames) | |
| end_idx = start_idx + mel_step_size | |
| if start_idx >= T: | |
| m = np.zeros((mel.shape[0], mel_step_size), dtype=mel.dtype) | |
| elif end_idx > T: | |
| pad = end_idx - T | |
| m = np.pad(mel[:, start_idx:], ((0, 0), (0, pad)), mode='constant') | |
| elif start_idx < 0: | |
| pad = -start_idx | |
| take = mel[:, :max(0, end_idx)] | |
| m = np.pad(take, ((0,0),(pad,0)), mode='constant') | |
| if m.shape[1] < mel_step_size: | |
| m = np.pad(m, ((0,0),(0, mel_step_size - m.shape[1])), mode='constant') | |
| else: | |
| m = mel[:, start_idx:end_idx] | |
| chunks.append(m) | |
| return chunks | |
| base_offset = int(args.mel_offset_frames) | |
| mel_chunks = build_mel_chunks(base_offset, mel_idx_multiplier) | |
| # Optional: auto calibration of global offset by correlating mouth openness with mel energy | |
| if (not args.static) and getattr(args, 'auto_sync_calibrate', False): | |
| try: | |
| # Compute mouth openness per frame from 68-point landmarks | |
| # Using pairs: (62,66), (63,67), (61,65) in 0-based indices | |
| def mouth_open_metric(lms): | |
| if lms is None or len(lms) < 68: | |
| return 0.0 | |
| p = lms | |
| pairs = [(62,66), (63,67), (61,65)] | |
| vals = [] | |
| for a,b in pairs: | |
| ya, yb = float(p[a][1]), float(p[b][1]) | |
| vals.append(abs(yb - ya)) | |
| return float(np.mean(vals)) if len(vals) > 0 else 0.0 | |
| mouth_series = np.array([mouth_open_metric(land) for land in lm[:len(full_frames)]], dtype=np.float32) | |
| def nz_norm(x): | |
| x = x - x.mean() | |
| s = x.std() | |
| return x / (s if s > 1e-6 else 1.0) | |
| m_norm = nz_norm(mouth_series) | |
| max_lag = int(getattr(args, 'calibrate_window', 6)) | |
| def evaluate_alignment(chunks): | |
| energy_series = np.array([float((mc * mc).mean()) for mc in chunks], dtype=np.float32) | |
| e_norm = nz_norm(energy_series) | |
| best_corr_local, best_off_local = -1e9, 0 | |
| for off in range(-max_lag, max_lag+1): | |
| if off >= 0: | |
| mn = m_norm[off:] | |
| en = e_norm[:len(m_norm)-off] | |
| else: | |
| mn = m_norm[:off] | |
| en = e_norm[-off:] | |
| L = min(len(mn), len(en)) | |
| if L < 10: | |
| continue | |
| c = float(np.corrcoef(mn[:L], en[:L])[0,1]) | |
| if c > best_corr_local: | |
| best_corr_local, best_off_local = c, off | |
| return best_corr_local, best_off_local | |
| best_global = -1e9 | |
| best_chunks = mel_chunks | |
| best_offset = 0 | |
| best_multiplier = mel_idx_multiplier | |
| scale_search = getattr(args, 'auto_sync_scale', False) | |
| if scale_search: | |
| scale_range = float(max(0.0, getattr(args, 'sync_scale_range', 0.08))) | |
| steps = max(1, int(getattr(args, 'sync_scale_steps', 5))) | |
| scales = np.linspace(1.0 - scale_range, 1.0 + scale_range, 2 * steps + 1) | |
| else: | |
| scales = [1.0] | |
| for scale in scales: | |
| candidate_multiplier = base_multiplier * scale | |
| candidate_chunks = build_mel_chunks(base_offset, candidate_multiplier) | |
| corr, off = evaluate_alignment(candidate_chunks) | |
| if corr > best_global: | |
| best_global = corr | |
| best_chunks = candidate_chunks | |
| best_offset = off | |
| best_multiplier = candidate_multiplier | |
| mel_idx_multiplier = best_multiplier | |
| if best_offset != 0: | |
| print(f"[Calibrate] Best offset {best_offset} frames (corr={best_global:.3f}). Rebuilding mel chunks.") | |
| base_offset += best_offset | |
| best_chunks = build_mel_chunks(base_offset, mel_idx_multiplier) | |
| if scale_search and abs(best_multiplier - base_multiplier) > 1e-6: | |
| scale_factor = best_multiplier / base_multiplier if base_multiplier != 0 else 1.0 | |
| print(f"[Calibrate] Applied scale factor {scale_factor:.4f} to mel/frame alignment.") | |
| mel_chunks = best_chunks | |
| except Exception as e: | |
| print(f"[Calibrate] Skipped due to error: {e}") | |
| print(f"✅ Built {len(mel_chunks)} mel chunks for {len(full_frames)} frames (multiplier={mel_idx_multiplier:.3f}, offset_frames={base_offset})") | |
| print("[Step 4] Load audio; Length of mel chunks: {}".format(len(mel_chunks))) | |
| imgs_enhanced = [] | |
| for idx in tqdm(range(len(imgs)), desc='[Step 5] Reference Enhancement'): | |
| img = imgs[idx] | |
| pred, _, _ = enhancer.process(img, img, face_enhance=True, possion_blending=False) | |
| imgs_enhanced.append(pred) | |
| gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2)) | |
| frame_h, frame_w = full_frames[0].shape[:-1] | |
| out = cv2.VideoWriter('temp/{}/result.mp4'.format(args.tmp_dir), cv2.VideoWriter_fourcc(*'mp4v'), fps, (frame_w, frame_h)) | |
| if args.up_face != 'original': | |
| instance = GANimationModel() | |
| instance.initialize() | |
| instance.setup() | |
| kp_extractor = KeypointExtractor() | |
| prev_lip_patch = None | |
| prev_face_gray = None | |
| temporal_buffer = [] | |
| temporal_window = max(1, int(getattr(args, 'lip_temporal_window', 5))) | |
| lip_mode = str(getattr(args, 'lip_temporal_mode', 'ema')).lower() | |
| # Optional warmup run to stabilize first frames | |
| if getattr(args, 'warmup_frames', 0) and args.warmup_frames > 0: | |
| try: | |
| warm_item = next(gen) | |
| img_batch, mel_batch, frames, coords, img_original, f_frames = warm_item | |
| img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) | |
| mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) | |
| img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255. | |
| with torch.no_grad(): | |
| incomplete, reference = torch.split(img_batch, 3, dim=1) | |
| _ = model(mel_batch, img_batch, reference) | |
| except StopIteration: | |
| pass | |
| # Recreate generator so we don't lose the first batch | |
| gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2)) | |
| for i, (img_batch, mel_batch, frames, coords, img_original, f_frames) in enumerate(tqdm(gen, desc='[Step 6] Lip Synthesis:', total=int(np.ceil(float(len(mel_chunks)) / args.LNet_batch_size)))): | |
| img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device) | |
| mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device) | |
| img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255. # BGR -> RGB | |
| # Validate mel batches to ensure minimum spatial dimensions | |
| valid_indices = [] | |
| for bi in range(mel_batch.shape[0]): | |
| sample = mel_batch[bi] | |
| if sample.shape[-1] < 3 or sample.shape[-2] < 3: | |
| print(f"⚠️ Skipping invalid mel chunk shape: {tuple(sample.shape)}") | |
| else: | |
| valid_indices.append(bi) | |
| if len(valid_indices) == 0: | |
| raise ValueError("❌ All mel chunks were invalid. Check audio length or hop_size.") | |
| if len(valid_indices) != mel_batch.shape[0]: | |
| mel_batch = mel_batch[valid_indices] | |
| img_batch = img_batch[valid_indices] | |
| img_original = img_original[valid_indices] | |
| # Also filter associated metadata to keep alignment | |
| frames = [frames[j] for j in valid_indices] | |
| coords = [coords[j] for j in valid_indices] | |
| f_frames = [f_frames[j] for j in valid_indices] | |
| print(f"✅ Proceeding with {mel_batch.shape[0]} valid mel chunks of shape {tuple(mel_batch.shape[-2:])}") | |
| latent_scale = max(0.0, float(getattr(args, 'latent_noise_scale', 0.0))) | |
| if latent_scale > 0.0: | |
| mel_batch = mel_batch + torch.randn_like(mel_batch) * latent_scale | |
| with torch.no_grad(): | |
| incomplete, reference = torch.split(img_batch, 3, dim=1) | |
| pred, low_res = model(mel_batch, img_batch, reference) | |
| pred = torch.clamp(pred, 0, 1) | |
| if args.up_face in ['sad', 'angry', 'surprise']: | |
| tar_aus = exp_aus_dict[args.up_face] | |
| else: | |
| pass | |
| if args.up_face == 'original': | |
| cur_gen_faces = img_original | |
| else: | |
| test_batch = {'src_img': torch.nn.functional.interpolate((img_original * 2 - 1), size=(128, 128), mode='bilinear'), | |
| 'tar_aus': tar_aus.repeat(len(incomplete), 1)} | |
| instance.feed_batch(test_batch) | |
| instance.forward() | |
| cur_gen_faces = torch.nn.functional.interpolate( | |
| instance.fake_img / 2. + 0.5, | |
| size=(args.img_size, args.img_size), | |
| mode='bilinear' | |
| ) | |
| if args.without_rl1 is not False: | |
| incomplete, reference = torch.split(img_batch, 3, dim=1) | |
| mask = torch.where(incomplete==0, torch.ones_like(incomplete), torch.zeros_like(incomplete)) | |
| pred = pred * mask + cur_gen_faces * (1 - mask) | |
| pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255. | |
| if args.lip_temporal_smooth: | |
| base_weight = min(max(args.temporal_blend_weight, 0.0), 1.0) | |
| override_smooth = getattr(args, 'mouth_region_smoothing', None) | |
| if override_smooth is not None: | |
| base_weight = float(min(max(override_smooth, 0.0), 1.0)) | |
| adaptive = getattr(args, 'temporal_blend_auto', False) | |
| k = float(getattr(args, 'temporal_blend_k', 5.0)) | |
| smoothed = [] | |
| # Compute per-sample energy from current mel_batch | |
| try: | |
| mel_energy = (mel_batch ** 2).mean(dim=[1,2,3]).detach().cpu().numpy() | |
| e_min, e_max = float(mel_energy.min()), float(mel_energy.max()) | |
| denom = (e_max - e_min) if (e_max - e_min) > 1e-6 else 1.0 | |
| mel_energy_n = (mel_energy - e_min) / denom | |
| except Exception: | |
| adaptive = False | |
| mel_energy_n = None | |
| # Prepare SyncNet or proxy confidence support | |
| sync_gate = bool(getattr(args, 'enable_syncnet_gate', False)) | |
| conf_thr = float(getattr(args, 'lip_sync_conf_threshold', 0.0)) | |
| use_gate = sync_gate and conf_thr > 0.0 | |
| sync_model = None | |
| if use_gate: | |
| sync_model = _maybe_load_syncnet(getattr(args, 'syncnet_model_path', 'checkpoints/syncnet.pt')) | |
| for idx, p in enumerate(pred): | |
| out_patch = p | |
| # EMA smoothing using previous frame | |
| if lip_mode in ['ema', 'both']: | |
| if prev_lip_patch is not None: | |
| if adaptive and mel_energy_n is not None: | |
| w = base_weight * float(np.exp(-k * mel_energy_n[idx])) | |
| else: | |
| w = base_weight | |
| w = float(min(max(w, 0.0), 1.0)) | |
| out_patch = (1.0 - w) * p + w * prev_lip_patch | |
| # Optical-flow guided warping of previous patch to current geometry | |
| if getattr(args, 'flow_temporal_smooth', False) and prev_lip_patch is not None: | |
| try: | |
| # Prepare current face gray image from original face patch | |
| cur_face_tensor = img_original[idx] # shape: 3xH xW (RGB, 0..1) | |
| cur_face = (cur_face_tensor.detach().cpu().numpy().transpose(1, 2, 0) * 255.).astype(np.uint8) | |
| cur_gray = cv2.cvtColor(cur_face, cv2.COLOR_RGB2GRAY) | |
| if prev_face_gray is not None and prev_face_gray.shape == cur_gray.shape: | |
| flow = cv2.calcOpticalFlowFarneback( | |
| prev_face_gray, cur_gray, None, | |
| float(getattr(args, 'flow_pyr_scale', 0.5)), | |
| int(getattr(args, 'flow_levels', 3)), | |
| int(getattr(args, 'flow_winsize', 21)), | |
| int(getattr(args, 'flow_iters', 3)), | |
| int(getattr(args, 'flow_poly_n', 5)), | |
| float(getattr(args, 'flow_poly_sigma', 1.2)), | |
| 0 | |
| ) | |
| h, w = cur_gray.shape | |
| grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h)) | |
| map_x = (grid_x + flow[..., 0]).astype(np.float32) | |
| map_y = (grid_y + flow[..., 1]).astype(np.float32) | |
| warped_prev = cv2.remap(prev_lip_patch.astype(np.float32), map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT) | |
| wf = float(getattr(args, 'flow_blend_weight', 0.4)) | |
| if override_smooth is not None: | |
| wf = float(min(max(override_smooth, 0.0), 1.0)) | |
| wf = max(0.0, min(1.0, wf)) | |
| out_patch = (1.0 - wf) * out_patch + wf * warped_prev | |
| # update prev_face_gray for next iteration | |
| prev_face_gray = cur_gray | |
| except Exception: | |
| # if any failure in flow, just skip silently | |
| pass | |
| # Median over a small temporal window to remove flicker | |
| if lip_mode in ['median', 'both']: | |
| temporal_buffer.append(out_patch) | |
| if len(temporal_buffer) > temporal_window: | |
| temporal_buffer.pop(0) | |
| stack = np.stack(temporal_buffer, axis=0) | |
| out_patch = np.median(stack, axis=0) | |
| # Sync confidence gating: hold more of previous patch if confidence is low | |
| if use_gate and prev_lip_patch is not None: | |
| try: | |
| if sync_model is not None: | |
| # Placeholder: if a TorchScript syncnet returns a scalar conf for current sample | |
| # Users can swap this with their actual model interface | |
| # Example expects input shapes to match model requirements; adjust as needed | |
| with torch.no_grad(): | |
| # Provide minimal placeholders; real implementation should pass mouth ROI and mel window | |
| conf_tensor = torch.tensor([float(mel_energy_n[idx])], dtype=torch.float32) | |
| conf = float(conf_tensor.item()) | |
| else: | |
| # Proxy confidence: normalized mel energy for this frame | |
| conf = float(mel_energy_n[idx]) if mel_energy_n is not None else 1.0 | |
| if conf < conf_thr: | |
| hold_w = min(0.95, max(0.5, 0.5 + (conf_thr - conf))) | |
| out_patch = hold_w * prev_lip_patch + (1.0 - hold_w) * out_patch | |
| except Exception: | |
| pass | |
| smoothed.append(out_patch) | |
| prev_lip_patch = out_patch | |
| pred = np.stack(smoothed, axis=0) | |
| torch.cuda.empty_cache() | |
| for p, f, xf, c in zip(pred, frames, f_frames, coords): | |
| y1, y2, x1, x2 = c | |
| p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1)) | |
| ff = xf.copy() | |
| ff[y1:y2, x1:x2] = p | |
| # month region enhancement by GFPGAN | |
| cropped_faces, restored_faces, restored_img = restorer.enhance( | |
| ff, has_aligned=False, only_center_face=True, paste_back=True) | |
| # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, | |
| mm = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0] | |
| mouse_mask = np.zeros_like(restored_img) | |
| tmp_mask = enhancer.faceparser.process(restored_img[y1:y2, x1:x2], mm)[0] | |
| tmp_mask = mask_postprocess(tmp_mask, thres=getattr(args, 'mask_feather', 20)) | |
| mouse_mask[y1:y2, x1:x2]= cv2.resize(tmp_mask, (x2 - x1, y2 - y1))[:, :, np.newaxis] / 255. | |
| height, width = ff.shape[:2] | |
| # Ensure shapes align for blending at native resolution | |
| restored_img = restored_img if restored_img.shape[:2] == (height, width) else cv2.resize(restored_img, (width, height)) | |
| ff = ff if ff.shape[:2] == (height, width) else cv2.resize(ff, (width, height)) | |
| full_mask = np.float32(mouse_mask) | |
| if full_mask.shape[:2] != (height, width): | |
| full_mask = cv2.resize(full_mask, (width, height)) | |
| # --- Shape alignment fix for Laplacian blending --- | |
| if restored_img.shape != ff.shape: | |
| ff = cv2.resize(ff, (restored_img.shape[1], restored_img.shape[0])) | |
| print(f"[Fix] Resized ff to {ff.shape} to match restored_img") | |
| if full_mask.shape[:2] != restored_img.shape[:2]: | |
| full_mask = cv2.resize(full_mask, (restored_img.shape[1], restored_img.shape[0])) | |
| print(f"[Fix] Resized full_mask to {full_mask.shape} to match restored_img") | |
| print("Shapes before blending:", restored_img.shape, ff.shape, full_mask.shape) | |
| img = Laplacian_Pyramid_Blending_with_mask(restored_img, ff, full_mask[:, :, 0], 12) | |
| img = np.clip(img, 0, 255) | |
| if img.shape[:2] != (height, width): | |
| pp = np.uint8(cv2.resize(img, (width, height))) | |
| else: | |
| pp = np.uint8(img) | |
| pp, orig_faces, enhanced_faces = enhancer.process(pp, xf, bbox=c, face_enhance=True, possion_blending=False) | |
| out.write(pp) | |
| out.release() | |
| if not os.path.isdir(os.path.dirname(args.outfile)): | |
| os.makedirs(os.path.dirname(args.outfile), exist_ok=True) | |
| if platform.system() == 'Windows': | |
| command = ['ffmpeg', '-loglevel', 'error', '-y', '-i', args.audio, '-i', 'temp/{}/result.mp4'.format(args.tmp_dir), '-shortest', '-c:v', 'copy', args.outfile] | |
| subprocess.call(command, shell=False) | |
| else: | |
| command = 'ffmpeg -loglevel error -y -i {} -i {} -shortest -c:v copy {}'.format(args.audio, 'temp/{}/result.mp4'.format(args.tmp_dir), args.outfile) | |
| subprocess.call(command, shell=True) | |
| print('outfile:', args.outfile) | |
| # frames:256x256, full_frames: original size | |
| def datagen(frames, mels, full_frames, frames_pil, cox): | |
| img_batch, mel_batch, frame_batch, coords_batch, ref_batch, full_frame_batch = [], [], [], [], [], [] | |
| base_name = args.face.split('/')[-1] | |
| refs = [] | |
| image_size = 256 | |
| # original frames | |
| kp_extractor = KeypointExtractor() | |
| fr_pil = [Image.fromarray(frame) for frame in frames] | |
| lms = kp_extractor.extract_keypoint(fr_pil, 'temp/'+base_name+'x12_landmarks.txt') | |
| frames_pil = [ (lm, frame) for frame,lm in zip(fr_pil, lms)] # frames is the croped version of modified face | |
| crops, orig_images, quads = crop_faces(image_size, frames_pil, scale=1.0, use_fa=True) | |
| # Guard against invalid (NaN/inf) quads by reusing last valid or falling back to identity | |
| clean_quads = [] | |
| identity_quad = np.array([[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]], dtype=np.float32) | |
| for q in quads: | |
| if q is None or (not np.isfinite(q).all()): | |
| q = clean_quads[-1] if len(clean_quads) > 0 else identity_quad | |
| clean_quads.append(q) | |
| # EMA smooth the alignment quads to reduce temporal jitter in warping | |
| beta_q = float(getattr(args, 'quad_ema_beta', 0.0)) | |
| if beta_q > 0.0 and len(clean_quads) > 1: | |
| sm_quads = [] | |
| prev_q = clean_quads[0].astype(np.float32) | |
| sm_quads.append(prev_q) | |
| for q in clean_quads[1:]: | |
| qf = q.astype(np.float32) | |
| prev_q = beta_q * prev_q + (1.0 - beta_q) * qf | |
| sm_quads.append(prev_q) | |
| else: | |
| sm_quads = clean_quads | |
| inverse_transforms = [calc_alignment_coefficients(q + 0.5, [[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]]) for q in sm_quads] | |
| del kp_extractor.detector | |
| oy1,oy2,ox1,ox2 = cox | |
| face_det_results = face_detect(full_frames, args, jaw_correction=True) | |
| for inverse_transform, crop, full_frame, face_det in zip(inverse_transforms, crops, full_frames, face_det_results): | |
| imc_pil = paste_image(inverse_transform, crop, Image.fromarray( | |
| cv2.resize(full_frame[int(oy1):int(oy2), int(ox1):int(ox2)], (256, 256)))) | |
| ff = full_frame.copy() | |
| ff[int(oy1):int(oy2), int(ox1):int(ox2)] = cv2.resize(np.array(imc_pil.convert('RGB')), (ox2 - ox1, oy2 - oy1)) | |
| oface, coords = face_det | |
| y1, y2, x1, x2 = coords | |
| refs.append(ff[y1: y2, x1:x2]) | |
| for i, m in enumerate(mels): | |
| idx = 0 if args.static else i | |
| frame_to_save = frames[idx].copy() | |
| face = refs[idx] | |
| oface, coords = face_det_results[idx].copy() | |
| face = cv2.resize(face, (args.img_size, args.img_size)) | |
| oface = cv2.resize(oface, (args.img_size, args.img_size)) | |
| img_batch.append(oface) | |
| ref_batch.append(face) | |
| mel_batch.append(m) | |
| coords_batch.append(coords) | |
| frame_batch.append(frame_to_save) | |
| full_frame_batch.append(full_frames[idx].copy()) | |
| if len(img_batch) >= args.LNet_batch_size: | |
| img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch) | |
| img_masked = img_batch.copy() | |
| img_original = img_batch.copy() | |
| img_masked[:, args.img_size//2:] = 0 | |
| img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255. | |
| mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) | |
| yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch | |
| img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch, ref_batch = [], [], [], [], [], [], [] | |
| if len(img_batch) > 0: | |
| img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch) | |
| img_masked = img_batch.copy() | |
| img_original = img_batch.copy() | |
| img_masked[:, args.img_size//2:] = 0 | |
| img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255. | |
| mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1]) | |
| yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch | |
| if __name__ == '__main__': | |
| main() | |