VideoReTalk / inference.py
Anirudh1950's picture
Publish project from GitHub
f80a40e verified
Raw History Blame Contribute Delete
52.4 kB
import numpy as np
import cv2, os, sys, subprocess, platform, torch
import math
from tqdm import tqdm
from PIL import Image
from scipy.io import loadmat
sys.path.insert(0, 'third_part')
sys.path.insert(0, 'third_part/GPEN')
sys.path.insert(0, 'third_part/GFPGAN')
# 3dmm extraction
from third_part.face3d.util.preprocess import align_img
from third_part.face3d.util.load_mats import load_lm3d
from third_part.face3d.extract_kp_videos import KeypointExtractor
# face enhancement
from third_part.GPEN.gpen_face_enhancer import FaceEnhancement
from third_part.GFPGAN.gfpgan import GFPGANer
# expression control
from third_part.ganimation_replicate.model.ganimation import GANimationModel
from utils import audio
from utils.ffhq_preprocess import Croper
from utils.alignment_stit import crop_faces, calc_alignment_coefficients, paste_image
from utils.inference_utils import Laplacian_Pyramid_Blending_with_mask, face_detect, load_model, options, split_coeff, \
trans_image, transform_semantic, find_crop_norm_ratio, load_face3d_net, exp_aus_dict, mask_postprocess
import warnings
warnings.filterwarnings("ignore")
args = options()
# Optional: attempt to load a SyncNet model if provided; otherwise fall back to proxy confidence
_syncnet = None
_syncnet_warned = False
def _maybe_load_syncnet(path):
global _syncnet, _syncnet_warned
if _syncnet is not None:
return _syncnet
try:
if os.path.isfile(path):
# Expect a TorchScript or torch.load-able module that exposes forward(audio, video)->embeds or confidence
_syncnet = torch.jit.load(path, map_location='cpu').eval()
else:
if not _syncnet_warned:
print(f"[SyncNet] Model not found at {path}. Using proxy confidence.")
_syncnet_warned = True
_syncnet = None
except Exception as e:
if not _syncnet_warned:
print(f"[SyncNet] Failed to load model ({e}). Using proxy confidence.")
_syncnet_warned = True
_syncnet = None
return _syncnet
def _ffprobe_duration(path):
try:
cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nk=1:nw=1', path]
out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return float(out.decode().strip())
except Exception as e:
print(f"[Warn] ffprobe failed for {path}: {e}")
return None
def _split_into_chunks(face_path, audio_path, out_dir, fps, chunk_duration, overlap, enforce_sr=16000, crossfade=False):
os.makedirs(out_dir, exist_ok=True)
video_dur = _ffprobe_duration(face_path) or 0
audio_dur = _ffprobe_duration(audio_path) or 0
total_dur = min(video_dur, audio_dur) if (video_dur > 0 and audio_dur > 0) else max(video_dur, audio_dur)
if total_dur == 0:
raise RuntimeError('Could not determine media duration')
eff_overlap = overlap if crossfade else 0.0
step = max(0.0, chunk_duration - eff_overlap) if chunk_duration > 0 else total_dur
starts = []
t = 0.0
while t < total_dur:
starts.append(t)
t += step if step > 0 else total_dur
chunk_paths = []
for idx, start in enumerate(starts):
dur = min(chunk_duration if chunk_duration > 0 else total_dur, total_dur - start)
v_out = os.path.join(out_dir, f'video_{idx+1:03d}.mp4')
a_out = os.path.join(out_dir, f'audio_{idx+1:03d}.wav')
# Re-encode video chunk to enforce exact boundaries & fps
vcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', face_path, '-t', f'{dur:.3f}',
'-r', f'{fps:.3f}', '-an', '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18', '-pix_fmt', 'yuv420p', v_out]
acmd = ['ffmpeg', '-loglevel', 'error', '-y', '-ss', f'{start:.3f}', '-i', audio_path, '-t', f'{dur:.3f}',
'-ac', '1', '-ar', str(enforce_sr), a_out]
try:
subprocess.check_call(vcmd, shell=False)
subprocess.check_call(acmd, shell=False)
except subprocess.CalledProcessError as e:
print(f"[Warn] Failed to split chunk {idx+1}: {e}")
continue
# Validate audio sample rate
try:
sr = _probe_audio_sr(a_out)
if sr != enforce_sr:
print(f"[Warn] Fixing sample rate for {a_out} (got {sr}); forcing {enforce_sr}")
fixcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', a_out, '-ac', '1', '-ar', str(enforce_sr), a_out]
subprocess.check_call(fixcmd, shell=False)
except Exception as e:
print(f"[Warn] Could not validate audio sr for {a_out}: {e}")
chunk_paths.append((v_out, a_out))
return chunk_paths
def _probe_audio_sr(path):
cmd = ['ffprobe', '-v', 'error', '-select_streams', 'a:0', '-show_entries', 'stream=sample_rate', '-of', 'default=nk=1:nw=1', path]
out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return int(out.decode().strip())
def _concat_media(list_file, output_path, stream='video', reencode=False, drop_audio=False):
# stream: 'video' or 'audio'
if stream == 'video':
if reencode:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c:v', 'libx264', '-pix_fmt', 'yuv420p']
else:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy']
if drop_audio:
cmd += ['-an']
cmd += [output_path]
else:
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list_file, '-c', 'copy', output_path]
subprocess.check_call(cmd, shell=False)
def _xfade_concatenate(videos, out_path, overlap):
# Simple pairwise xfade across N segments
if len(videos) == 1:
return videos[0]
cur = videos[0]
for i in range(1, len(videos)):
nxt = videos[i]
tmp = out_path.replace('.mp4', f'_xf{i:03d}.mp4')
cmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', cur, '-i', nxt,
'-filter_complex', f"[0:v][1:v]xfade=transition=fade:duration={overlap:.3f}:offset=PTS-STARTPTS[v];[0:a][1:a]acrossfade=d={overlap:.3f}[a]",
'-map', '[v]', '-map', '[a]', tmp]
subprocess.check_call(cmd, shell=False)
cur = tmp
os.replace(cur, out_path)
return out_path
# --- Video stabilization helpers ---
def _stabilize_video_with_vidstab(input_path, out_dir, shakiness=5, smoothing=30, crop_mode='keep'):
os.makedirs(out_dir, exist_ok=True)
trf_path = os.path.join(out_dir, 'vidstab_transform.trf')
stabilized_path = os.path.join(out_dir, 'stabilized_input.mp4')
# Normalize paths for ffmpeg filters
def _norm(p):
return os.path.abspath(p).replace('\\', '/')
input_norm = _norm(input_path)
trf_norm = _norm(trf_path)
# Pass 1: detect
detect_filter = f"vidstabdetect=shakiness={int(shakiness)}:accuracy=15:result={trf_norm}"
detect_cmd = ['ffmpeg', '-loglevel', 'error', '-y',
'-i', input_norm,
'-vf', detect_filter,
'-f', 'null', '-']
# Pass 2: transform
crop_mode = crop_mode if crop_mode in ['keep', 'content', 'black'] else 'keep'
transform_filter = f"vidstabtransform=smoothing={int(smoothing)}:input={trf_norm}:crop={crop_mode}"
transform_cmd = ['ffmpeg', '-loglevel', 'error', '-y',
'-i', input_norm,
'-vf', transform_filter,
'-c:v', 'libx264', '-preset', 'medium', '-crf', '18',
'-c:a', 'copy',
_norm(stabilized_path)]
try:
subprocess.check_call(detect_cmd, shell=False)
subprocess.check_call(transform_cmd, shell=False)
except subprocess.CalledProcessError as e:
raise RuntimeError(f'ffmpeg vidstab failed: {e}')
return stabilized_path
# --- Sync gating helpers ---
def _mouth_open_metric(lms):
try:
if lms is None or len(lms) < 68:
return 0.0
p = lms
pairs = [(62,66), (63,67), (61,65)]
vals = []
for a, b in pairs:
ya, yb = float(p[a][1]), float(p[b][1])
vals.append(abs(yb - ya))
return float(np.mean(vals)) if len(vals) > 0 else 0.0
except Exception:
return 0.0
def _sync_confidence(frame_idx, lm_all, mel_chunks, win=5):
try:
n = len(mel_chunks)
if n == 0:
return 0.0
s = max(0, frame_idx - win//2)
e = min(n, frame_idx + (win - win//2))
if e - s < 3:
return 0.0
mouth_series = np.array([_mouth_open_metric(lm_all[k]) for k in range(s, e)], dtype=np.float32)
energy_series = np.array([float((mel_chunks[k].astype(np.float32) ** 2).mean()) for k in range(s, e)], dtype=np.float32)
# normalize
def nz_norm(x):
x = x - x.mean()
sdev = x.std()
return x / (sdev if sdev > 1e-6 else 1.0)
m_norm = nz_norm(mouth_series)
e_norm = nz_norm(energy_series)
L = min(len(m_norm), len(e_norm))
if L < 3:
return 0.0
corr = float(np.corrcoef(m_norm[:L], e_norm[:L])[0, 1])
return max(0.0, min(1.0, 0.5 * (corr + 1.0)))
except Exception:
return 0.0
def main():
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('[Info] Using {} for inference.'.format(device))
os.makedirs(os.path.join('temp', args.tmp_dir), exist_ok=True)
enhancer = FaceEnhancement(base_dir='checkpoints', size=args.gpen_size, model=args.gpen_model, use_sr=False, \
sr_model='rrdb_realesrnet_psnr', channel_multiplier=2, narrow=1, device=device)
restorer = GFPGANer(model_path='checkpoints/GFPGANv1.3.pth', upscale=1, arch='clean', \
channel_multiplier=2, bg_upsampler=None)
base_name = args.face.split('/')[-1]
if os.path.isfile(args.face) and args.face.split('.')[1] in ['jpg', 'png', 'jpeg']:
args.static = True
# Enforce one-shot for static images
if not args.one_shot:
print('[Info] Static image detected; enabling one-shot mode.')
args.one_shot = True
if not os.path.isfile(args.face):
raise ValueError('--face argument must be a valid path to video/image file')
elif args.face.split('.')[1] in ['jpg', 'png', 'jpeg']:
full_frames = [cv2.imread(args.face)]
fps = args.fps
else:
# Disable one-shot for videos
if args.one_shot:
print('[Info] Video detected; disabling one-shot mode (only for static images).')
args.one_shot = False
if getattr(args, 'pre_stabilize_video', False):
try:
stabilize_dir = os.path.join('temp', args.tmp_dir, 'stabilize')
stabilized_face = _stabilize_video_with_vidstab(
args.face,
stabilize_dir,
shakiness=getattr(args, 'stabilize_shakiness', 5),
smoothing=getattr(args, 'stabilize_smoothing', 30),
crop_mode=getattr(args, 'stabilize_crop', 'keep')
)
print(f"[Stabilize] Using stabilized video at {stabilized_face}")
args.face = stabilized_face
except Exception as e:
print(f"[Stabilize] Warning: stabilization failed ({e}). Proceeding with original video.")
# Chunked processing for long videos
if args.enable_chunking and args.chunk_duration and args.chunk_duration > 0:
try:
video_dur = _ffprobe_duration(args.face) or 0
if video_dur > args.chunk_duration + 0.5:
# Probe fps
cap_tmp = cv2.VideoCapture(args.face)
fps_probe = cap_tmp.get(cv2.CAP_PROP_FPS)
cap_tmp.release()
if not fps_probe or math.isclose(fps_probe, 0.0) or np.isnan(fps_probe):
fps_probe = args.fps
parts_dir = os.path.join('temp', args.tmp_dir, args.chunk_temp_dir)
out_parts_dir = os.path.join('temp', args.tmp_dir, 'out_parts')
os.makedirs(parts_dir, exist_ok=True)
os.makedirs(out_parts_dir, exist_ok=True)
print(f"[Chunk] Splitting into ~{args.chunk_duration}s chunks with overlap={args.chunk_overlap}s")
chunks = _split_into_chunks(args.face, args.audio, parts_dir, fps_probe, args.chunk_duration, args.chunk_overlap, enforce_sr=16000, crossfade=args.chunk_crossfade)
# Minimal required seconds for a mel window
from utils import audio as _audio
min_sec = (args.mel_step_size * _audio.hp.hop_size) / _audio.hp.sample_rate
successes = []
for ci, (vpath, apath) in enumerate(chunks, start=1):
try:
# Validate length
c_dur = _ffprobe_duration(vpath) or 0
if c_dur < min_sec:
print(f"[Warn] Skipping chunk {ci:03d} too short: {c_dur:.3f}s < {min_sec:.3f}s")
continue
out_path = os.path.join(out_parts_dir, f'output_{ci:03d}.mp4')
cmd = [sys.executable, os.path.abspath(__file__), '--face', vpath, '--audio', apath, '--outfile', out_path, '--fps', f'{fps_probe:.3f}', '--tmp_dir', f"{args.tmp_dir}/chunk_{ci:03d}", '--chunk_duration', '0']
# Propagate useful flags
if args.audio_preprocess: cmd.append('--audio_preprocess')
if args.audio_normalize: cmd.append('--audio_normalize')
if args.audio_denoise: cmd.append('--audio_denoise')
if args.lip_temporal_smooth: cmd.append('--lip_temporal_smooth')
# Always re-preprocess per chunk to avoid stale landmark/cached files mismatch
cmd.append('--re_preprocess')
try:
subprocess.check_call(cmd, shell=False)
successes.append(out_path)
except subprocess.CalledProcessError as e:
print(f"[Warn] Chunk {ci:03d} failed: {e}")
continue
except Exception as e:
print(f"[Warn] Exception on chunk {ci:03d}: {e}")
continue
if len(successes) == 0:
raise RuntimeError('No chunk processed successfully')
final_out = args.outfile
if args.chunk_crossfade:
# Build crossfaded concatenation including audio
concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4')
_xfade_concatenate(successes, concat_video, max(0.0, args.chunk_overlap))
# Just move to final (already has audio)
if not os.path.isdir(os.path.dirname(final_out)):
os.makedirs(os.path.dirname(final_out), exist_ok=True)
if platform.system() == 'Windows':
subprocess.check_call(['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-c', 'copy', final_out], shell=False)
else:
subprocess.check_call(f'ffmpeg -loglevel error -y -i "{concat_video}" -c copy "{final_out}"', shell=True)
else:
# Concatenate videos (drop audio)
concat_v_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_v.txt')
with open(concat_v_list, 'w', encoding='utf-8') as f:
for p in successes:
_safe = os.path.abspath(p).replace('\\', '/')
f.write(f"file '{_safe}'\n")
concat_video = os.path.join('temp', args.tmp_dir, 'concat_video.mp4')
_concat_media(concat_v_list, concat_video, stream='video', reencode=False, drop_audio=True)
# Concatenate audios
chunk_audios = [os.path.join(parts_dir, f) for f in sorted(os.listdir(parts_dir)) if f.lower().startswith('audio_') and f.lower().endswith('.wav')]
concat_a_list = os.path.join('temp', args.tmp_dir, 'out_parts', 'concat_a.txt')
with open(concat_a_list, 'w', encoding='utf-8') as f:
for p in chunk_audios:
_safe = os.path.abspath(p).replace('\\', '/')
f.write(f"file '{_safe}'\n")
concat_audio = os.path.join('temp', args.tmp_dir, 'concat_audio.wav')
_concat_media(concat_a_list, concat_audio, stream='audio', reencode=False)
# Merge A/V with -shortest
if not os.path.isdir(os.path.dirname(final_out)):
os.makedirs(os.path.dirname(final_out), exist_ok=True)
if platform.system() == 'Windows':
mcmd = ['ffmpeg', '-loglevel', 'error', '-y', '-i', concat_video, '-i', concat_audio, '-shortest', '-c:v', 'copy', final_out]
subprocess.call(mcmd, shell=False)
else:
mcmd = f'ffmpeg -loglevel error -y -i "{concat_video}" -i "{concat_audio}" -shortest -c:v copy "{final_out}"'
subprocess.call(mcmd, shell=True)
print('outfile:', final_out)
return
except Exception as e:
print(f"[Warn] Chunked processing fallback due to error: {e}")
video_stream = cv2.VideoCapture(args.face)
fps = video_stream.get(cv2.CAP_PROP_FPS)
full_frames = []
while True:
still_reading, frame = video_stream.read()
if not still_reading:
video_stream.release()
break
y1, y2, x1, x2 = args.crop
if x2 == -1: x2 = frame.shape[1]
if y2 == -1: y2 = frame.shape[0]
frame = frame[y1:y2, x1:x2]
full_frames.append(frame)
print ("[Step 0] Number of frames available for inference: "+str(len(full_frames)))
# face detection & cropping, cropping the first frame as the style of FFHQ
croper = Croper('checkpoints/shape_predictor_68_face_landmarks.dat')
full_frames_RGB = [cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in full_frames]
full_frames_RGB, crop, quad = croper.crop(full_frames_RGB, xsize=512)
clx, cly, crx, cry = crop
lx, ly, rx, ry = quad
lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)
oy1, oy2, ox1, ox2 = cly+ly, min(cly+ry, full_frames[0].shape[0]), clx+lx, min(clx+rx, full_frames[0].shape[1])
# original_size = (ox2 - ox1, oy2 - oy1)
frames_pil = [Image.fromarray(cv2.resize(frame,(256,256))) for frame in full_frames_RGB]
# get the landmark according to the detected face.
if not os.path.isfile('temp/'+base_name+'_landmarks.txt') or args.re_preprocess:
print('[Step 1] Landmarks Extraction in Video.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
print('[Step 1] Using saved landmarks.')
lm_path = 'temp/'+base_name+'_landmarks.txt'
try:
lm_raw = np.loadtxt(lm_path).astype(np.float32)
except Exception as e:
print(f"[Step 1] Failed to load saved landmarks ({e}); re-extracting.")
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
total_vals = lm_raw.size
denom = 2 * len(full_frames)
if denom > 0 and total_vals % denom == 0:
k = total_vals // denom
try:
lm = lm_raw.reshape([len(full_frames), k, 2])
except Exception:
print('[Step 1] Saved landmarks shape mismatch; re-extracting.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
else:
print('[Step 1] Saved landmarks length does not match frames; re-extracting.')
kp_extractor = KeypointExtractor()
lm = kp_extractor.extract_keypoint(frames_pil, './temp/'+base_name+'_landmarks.txt')
if not os.path.isfile('temp/'+base_name+'_coeffs.npy') or args.exp_img is not None or args.re_preprocess:
net_recon = load_face3d_net(args.face3d_net_path, device)
lm3d_std = load_lm3d('checkpoints/BFM')
video_coeffs = []
for idx in tqdm(range(len(frames_pil)), desc="[Step 2] 3DMM Extraction In Video:"):
frame = frames_pil[idx]
W, H = frame.size
lm_idx = lm[idx].reshape([-1, 2])
if np.mean(lm_idx) == -1:
lm_idx = (lm3d_std[:, :2]+1) / 2.
lm_idx = np.concatenate([lm_idx[:, :1] * W, lm_idx[:, 1:2] * H], 1)
else:
lm_idx[:, -1] = H - 1 - lm_idx[:, -1]
trans_params, im_idx, lm_idx, _ = align_img(frame, lm_idx, lm3d_std)
trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
im_idx_tensor = torch.tensor(np.array(im_idx)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0)
with torch.no_grad():
coeffs = split_coeff(net_recon(im_idx_tensor))
pred_coeff = {key:coeffs[key].cpu().numpy() for key in coeffs}
pred_coeff = np.concatenate([pred_coeff['id'], pred_coeff['exp'], pred_coeff['tex'], pred_coeff['angle'],\
pred_coeff['gamma'], pred_coeff['trans'], trans_params[None]], 1)
video_coeffs.append(pred_coeff)
semantic_npy = np.array(video_coeffs)[:,0]
np.save('temp/'+base_name+'_coeffs.npy', semantic_npy)
else:
print('[Step 2] Using saved coeffs.')
semantic_npy = np.load('temp/'+base_name+'_coeffs.npy').astype(np.float32)
# generate the 3dmm coeff from a single image
if args.exp_img is not None and ('.png' in args.exp_img or '.jpg' in args.exp_img):
print('extract the exp from',args.exp_img)
exp_pil = Image.open(args.exp_img).convert('RGB')
lm3d_std = load_lm3d('third_part/face3d/BFM')
W, H = exp_pil.size
kp_extractor = KeypointExtractor()
lm_exp = kp_extractor.extract_keypoint([exp_pil], 'temp/'+base_name+'_temp.txt')[0]
if np.mean(lm_exp) == -1:
lm_exp = (lm3d_std[:, :2] + 1) / 2.
lm_exp = np.concatenate(
[lm_exp[:, :1] * W, lm_exp[:, 1:2] * H], 1)
else:
lm_exp[:, -1] = H - 1 - lm_exp[:, -1]
trans_params, im_exp, lm_exp, _ = align_img(exp_pil, lm_exp, lm3d_std)
trans_params = np.array([float(item) for item in np.hsplit(trans_params, 5)]).astype(np.float32)
im_exp_tensor = torch.tensor(np.array(im_exp)/255., dtype=torch.float32).permute(2, 0, 1).to(device).unsqueeze(0)
# Ensure face3d net is loaded when deriving expression from an image
if 'net_recon' not in locals():
net_recon = load_face3d_net(args.face3d_net_path, device)
with torch.no_grad():
expression = split_coeff(net_recon(im_exp_tensor))['exp'][0]
del net_recon
elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'neutral':
print('using neutral expression')
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0]
elif isinstance(args.exp_img, str) and args.exp_img.lower() == 'smile':
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_mouth'])[0]
else:
print('using neutral expression (default)')
expression = torch.tensor(loadmat('checkpoints/expression.mat')['expression_center'])[0]
# load DNet, model(LNet and ENet)
D_Net, model = load_model(args, device)
if not os.path.isfile('temp/'+base_name+'_stablized.npy') or args.re_preprocess:
imgs = []
for idx in tqdm(range(len(frames_pil)), desc="[Step 3] Stabilize the expression In Video:"):
if args.one_shot:
source_img = trans_image(frames_pil[0]).unsqueeze(0).to(device)
semantic_source_numpy = semantic_npy[0:1]
else:
source_img = trans_image(frames_pil[idx]).unsqueeze(0).to(device)
semantic_source_numpy = semantic_npy[idx:idx+1]
ratio = find_crop_norm_ratio(semantic_source_numpy, semantic_npy)
coeff = transform_semantic(semantic_npy, idx, ratio).unsqueeze(0).to(device)
# hacking the new expression
coeff[:, :64, :] = expression[None, :64, None].to(device)
with torch.no_grad():
output = D_Net(source_img, coeff)
img_stablized = np.uint8((output['fake_image'].squeeze(0).permute(1,2,0).cpu().clamp_(-1, 1).numpy() + 1 )/2. * 255)
imgs.append(cv2.cvtColor(img_stablized,cv2.COLOR_RGB2BGR))
np.save('temp/'+base_name+'_stablized.npy',imgs)
del D_Net
else:
print('[Step 3] Using saved stabilized video.')
imgs = np.load('temp/'+base_name+'_stablized.npy')
torch.cuda.empty_cache()
# Ensure 16kHz mono WAV and clean levels prior to mel extraction
processed_audio_path = 'temp/{}/audio_proc.wav'.format(args.tmp_dir)
if args.audio_preprocess or (not args.audio.endswith('.wav')):
afilters = []
if args.audio_denoise:
afilters.append('afftdn')
if args.audio_normalize:
afilters.append('dynaudnorm=f=150:g=15')
af_clause = ' -af "{}"'.format(','.join(afilters)) if len(afilters) > 0 else ''
command = 'ffmpeg -loglevel error -y -i "{}" -ac 1 -ar {}{} "{}"'.format(
args.audio, args.audio_target_sr, af_clause, processed_audio_path)
subprocess.call(command, shell=True)
args.audio = processed_audio_path
wav = audio.load_wav(args.audio, 16000)
mel = audio.melspectrogram(wav)
if np.isnan(mel.reshape(-1)).sum() > 0:
if args.audio_eps_on_nan:
# Add small epsilon noise (useful for some TTS outputs) and retry once
wav = wav + (1e-6 * np.random.randn(len(wav))).astype(wav.dtype)
mel = audio.melspectrogram(wav)
if np.isnan(mel.reshape(-1)).sum() > 0:
raise ValueError('Mel contains nan! Consider denoising/normalizing or adding epsilon noise and try again')
# Derive mel frames-per-second from audio hyperparams to stay in sync with hop_size
mel_frames_per_second = audio.hp.sample_rate / audio.hp.hop_size
mel_step_size = args.mel_step_size
base_multiplier = (mel_frames_per_second / fps) * args.mel_multiplier_scale
mel_idx_multiplier = base_multiplier
# Build one mel chunk per video frame (1:1 alignment)
def build_mel_chunks(offset_frames, multiplier):
chunks = []
T = mel.shape[1]
for frame_idx in range(len(full_frames)):
start_idx = int(round(frame_idx * multiplier)) + int(offset_frames)
end_idx = start_idx + mel_step_size
if start_idx >= T:
m = np.zeros((mel.shape[0], mel_step_size), dtype=mel.dtype)
elif end_idx > T:
pad = end_idx - T
m = np.pad(mel[:, start_idx:], ((0, 0), (0, pad)), mode='constant')
elif start_idx < 0:
pad = -start_idx
take = mel[:, :max(0, end_idx)]
m = np.pad(take, ((0,0),(pad,0)), mode='constant')
if m.shape[1] < mel_step_size:
m = np.pad(m, ((0,0),(0, mel_step_size - m.shape[1])), mode='constant')
else:
m = mel[:, start_idx:end_idx]
chunks.append(m)
return chunks
base_offset = int(args.mel_offset_frames)
mel_chunks = build_mel_chunks(base_offset, mel_idx_multiplier)
# Optional: auto calibration of global offset by correlating mouth openness with mel energy
if (not args.static) and getattr(args, 'auto_sync_calibrate', False):
try:
# Compute mouth openness per frame from 68-point landmarks
# Using pairs: (62,66), (63,67), (61,65) in 0-based indices
def mouth_open_metric(lms):
if lms is None or len(lms) < 68:
return 0.0
p = lms
pairs = [(62,66), (63,67), (61,65)]
vals = []
for a,b in pairs:
ya, yb = float(p[a][1]), float(p[b][1])
vals.append(abs(yb - ya))
return float(np.mean(vals)) if len(vals) > 0 else 0.0
mouth_series = np.array([mouth_open_metric(land) for land in lm[:len(full_frames)]], dtype=np.float32)
def nz_norm(x):
x = x - x.mean()
s = x.std()
return x / (s if s > 1e-6 else 1.0)
m_norm = nz_norm(mouth_series)
max_lag = int(getattr(args, 'calibrate_window', 6))
def evaluate_alignment(chunks):
energy_series = np.array([float((mc * mc).mean()) for mc in chunks], dtype=np.float32)
e_norm = nz_norm(energy_series)
best_corr_local, best_off_local = -1e9, 0
for off in range(-max_lag, max_lag+1):
if off >= 0:
mn = m_norm[off:]
en = e_norm[:len(m_norm)-off]
else:
mn = m_norm[:off]
en = e_norm[-off:]
L = min(len(mn), len(en))
if L < 10:
continue
c = float(np.corrcoef(mn[:L], en[:L])[0,1])
if c > best_corr_local:
best_corr_local, best_off_local = c, off
return best_corr_local, best_off_local
best_global = -1e9
best_chunks = mel_chunks
best_offset = 0
best_multiplier = mel_idx_multiplier
scale_search = getattr(args, 'auto_sync_scale', False)
if scale_search:
scale_range = float(max(0.0, getattr(args, 'sync_scale_range', 0.08)))
steps = max(1, int(getattr(args, 'sync_scale_steps', 5)))
scales = np.linspace(1.0 - scale_range, 1.0 + scale_range, 2 * steps + 1)
else:
scales = [1.0]
for scale in scales:
candidate_multiplier = base_multiplier * scale
candidate_chunks = build_mel_chunks(base_offset, candidate_multiplier)
corr, off = evaluate_alignment(candidate_chunks)
if corr > best_global:
best_global = corr
best_chunks = candidate_chunks
best_offset = off
best_multiplier = candidate_multiplier
mel_idx_multiplier = best_multiplier
if best_offset != 0:
print(f"[Calibrate] Best offset {best_offset} frames (corr={best_global:.3f}). Rebuilding mel chunks.")
base_offset += best_offset
best_chunks = build_mel_chunks(base_offset, mel_idx_multiplier)
if scale_search and abs(best_multiplier - base_multiplier) > 1e-6:
scale_factor = best_multiplier / base_multiplier if base_multiplier != 0 else 1.0
print(f"[Calibrate] Applied scale factor {scale_factor:.4f} to mel/frame alignment.")
mel_chunks = best_chunks
except Exception as e:
print(f"[Calibrate] Skipped due to error: {e}")
print(f"✅ Built {len(mel_chunks)} mel chunks for {len(full_frames)} frames (multiplier={mel_idx_multiplier:.3f}, offset_frames={base_offset})")
print("[Step 4] Load audio; Length of mel chunks: {}".format(len(mel_chunks)))
imgs_enhanced = []
for idx in tqdm(range(len(imgs)), desc='[Step 5] Reference Enhancement'):
img = imgs[idx]
pred, _, _ = enhancer.process(img, img, face_enhance=True, possion_blending=False)
imgs_enhanced.append(pred)
gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2))
frame_h, frame_w = full_frames[0].shape[:-1]
out = cv2.VideoWriter('temp/{}/result.mp4'.format(args.tmp_dir), cv2.VideoWriter_fourcc(*'mp4v'), fps, (frame_w, frame_h))
if args.up_face != 'original':
instance = GANimationModel()
instance.initialize()
instance.setup()
kp_extractor = KeypointExtractor()
prev_lip_patch = None
prev_face_gray = None
temporal_buffer = []
temporal_window = max(1, int(getattr(args, 'lip_temporal_window', 5)))
lip_mode = str(getattr(args, 'lip_temporal_mode', 'ema')).lower()
# Optional warmup run to stabilize first frames
if getattr(args, 'warmup_frames', 0) and args.warmup_frames > 0:
try:
warm_item = next(gen)
img_batch, mel_batch, frames, coords, img_original, f_frames = warm_item
img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device)
mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device)
img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255.
with torch.no_grad():
incomplete, reference = torch.split(img_batch, 3, dim=1)
_ = model(mel_batch, img_batch, reference)
except StopIteration:
pass
# Recreate generator so we don't lose the first batch
gen = datagen(imgs_enhanced.copy(), mel_chunks, full_frames, None, (oy1,oy2,ox1,ox2))
for i, (img_batch, mel_batch, frames, coords, img_original, f_frames) in enumerate(tqdm(gen, desc='[Step 6] Lip Synthesis:', total=int(np.ceil(float(len(mel_chunks)) / args.LNet_batch_size)))):
img_batch = torch.FloatTensor(np.transpose(img_batch, (0, 3, 1, 2))).to(device)
mel_batch = torch.FloatTensor(np.transpose(mel_batch, (0, 3, 1, 2))).to(device)
img_original = torch.FloatTensor(np.transpose(img_original, (0, 3, 1, 2))).to(device)/255. # BGR -> RGB
# Validate mel batches to ensure minimum spatial dimensions
valid_indices = []
for bi in range(mel_batch.shape[0]):
sample = mel_batch[bi]
if sample.shape[-1] < 3 or sample.shape[-2] < 3:
print(f"⚠️ Skipping invalid mel chunk shape: {tuple(sample.shape)}")
else:
valid_indices.append(bi)
if len(valid_indices) == 0:
raise ValueError("❌ All mel chunks were invalid. Check audio length or hop_size.")
if len(valid_indices) != mel_batch.shape[0]:
mel_batch = mel_batch[valid_indices]
img_batch = img_batch[valid_indices]
img_original = img_original[valid_indices]
# Also filter associated metadata to keep alignment
frames = [frames[j] for j in valid_indices]
coords = [coords[j] for j in valid_indices]
f_frames = [f_frames[j] for j in valid_indices]
print(f"✅ Proceeding with {mel_batch.shape[0]} valid mel chunks of shape {tuple(mel_batch.shape[-2:])}")
latent_scale = max(0.0, float(getattr(args, 'latent_noise_scale', 0.0)))
if latent_scale > 0.0:
mel_batch = mel_batch + torch.randn_like(mel_batch) * latent_scale
with torch.no_grad():
incomplete, reference = torch.split(img_batch, 3, dim=1)
pred, low_res = model(mel_batch, img_batch, reference)
pred = torch.clamp(pred, 0, 1)
if args.up_face in ['sad', 'angry', 'surprise']:
tar_aus = exp_aus_dict[args.up_face]
else:
pass
if args.up_face == 'original':
cur_gen_faces = img_original
else:
test_batch = {'src_img': torch.nn.functional.interpolate((img_original * 2 - 1), size=(128, 128), mode='bilinear'),
'tar_aus': tar_aus.repeat(len(incomplete), 1)}
instance.feed_batch(test_batch)
instance.forward()
cur_gen_faces = torch.nn.functional.interpolate(
instance.fake_img / 2. + 0.5,
size=(args.img_size, args.img_size),
mode='bilinear'
)
if args.without_rl1 is not False:
incomplete, reference = torch.split(img_batch, 3, dim=1)
mask = torch.where(incomplete==0, torch.ones_like(incomplete), torch.zeros_like(incomplete))
pred = pred * mask + cur_gen_faces * (1 - mask)
pred = pred.cpu().numpy().transpose(0, 2, 3, 1) * 255.
if args.lip_temporal_smooth:
base_weight = min(max(args.temporal_blend_weight, 0.0), 1.0)
override_smooth = getattr(args, 'mouth_region_smoothing', None)
if override_smooth is not None:
base_weight = float(min(max(override_smooth, 0.0), 1.0))
adaptive = getattr(args, 'temporal_blend_auto', False)
k = float(getattr(args, 'temporal_blend_k', 5.0))
smoothed = []
# Compute per-sample energy from current mel_batch
try:
mel_energy = (mel_batch ** 2).mean(dim=[1,2,3]).detach().cpu().numpy()
e_min, e_max = float(mel_energy.min()), float(mel_energy.max())
denom = (e_max - e_min) if (e_max - e_min) > 1e-6 else 1.0
mel_energy_n = (mel_energy - e_min) / denom
except Exception:
adaptive = False
mel_energy_n = None
# Prepare SyncNet or proxy confidence support
sync_gate = bool(getattr(args, 'enable_syncnet_gate', False))
conf_thr = float(getattr(args, 'lip_sync_conf_threshold', 0.0))
use_gate = sync_gate and conf_thr > 0.0
sync_model = None
if use_gate:
sync_model = _maybe_load_syncnet(getattr(args, 'syncnet_model_path', 'checkpoints/syncnet.pt'))
for idx, p in enumerate(pred):
out_patch = p
# EMA smoothing using previous frame
if lip_mode in ['ema', 'both']:
if prev_lip_patch is not None:
if adaptive and mel_energy_n is not None:
w = base_weight * float(np.exp(-k * mel_energy_n[idx]))
else:
w = base_weight
w = float(min(max(w, 0.0), 1.0))
out_patch = (1.0 - w) * p + w * prev_lip_patch
# Optical-flow guided warping of previous patch to current geometry
if getattr(args, 'flow_temporal_smooth', False) and prev_lip_patch is not None:
try:
# Prepare current face gray image from original face patch
cur_face_tensor = img_original[idx] # shape: 3xH xW (RGB, 0..1)
cur_face = (cur_face_tensor.detach().cpu().numpy().transpose(1, 2, 0) * 255.).astype(np.uint8)
cur_gray = cv2.cvtColor(cur_face, cv2.COLOR_RGB2GRAY)
if prev_face_gray is not None and prev_face_gray.shape == cur_gray.shape:
flow = cv2.calcOpticalFlowFarneback(
prev_face_gray, cur_gray, None,
float(getattr(args, 'flow_pyr_scale', 0.5)),
int(getattr(args, 'flow_levels', 3)),
int(getattr(args, 'flow_winsize', 21)),
int(getattr(args, 'flow_iters', 3)),
int(getattr(args, 'flow_poly_n', 5)),
float(getattr(args, 'flow_poly_sigma', 1.2)),
0
)
h, w = cur_gray.shape
grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))
map_x = (grid_x + flow[..., 0]).astype(np.float32)
map_y = (grid_y + flow[..., 1]).astype(np.float32)
warped_prev = cv2.remap(prev_lip_patch.astype(np.float32), map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
wf = float(getattr(args, 'flow_blend_weight', 0.4))
if override_smooth is not None:
wf = float(min(max(override_smooth, 0.0), 1.0))
wf = max(0.0, min(1.0, wf))
out_patch = (1.0 - wf) * out_patch + wf * warped_prev
# update prev_face_gray for next iteration
prev_face_gray = cur_gray
except Exception:
# if any failure in flow, just skip silently
pass
# Median over a small temporal window to remove flicker
if lip_mode in ['median', 'both']:
temporal_buffer.append(out_patch)
if len(temporal_buffer) > temporal_window:
temporal_buffer.pop(0)
stack = np.stack(temporal_buffer, axis=0)
out_patch = np.median(stack, axis=0)
# Sync confidence gating: hold more of previous patch if confidence is low
if use_gate and prev_lip_patch is not None:
try:
if sync_model is not None:
# Placeholder: if a TorchScript syncnet returns a scalar conf for current sample
# Users can swap this with their actual model interface
# Example expects input shapes to match model requirements; adjust as needed
with torch.no_grad():
# Provide minimal placeholders; real implementation should pass mouth ROI and mel window
conf_tensor = torch.tensor([float(mel_energy_n[idx])], dtype=torch.float32)
conf = float(conf_tensor.item())
else:
# Proxy confidence: normalized mel energy for this frame
conf = float(mel_energy_n[idx]) if mel_energy_n is not None else 1.0
if conf < conf_thr:
hold_w = min(0.95, max(0.5, 0.5 + (conf_thr - conf)))
out_patch = hold_w * prev_lip_patch + (1.0 - hold_w) * out_patch
except Exception:
pass
smoothed.append(out_patch)
prev_lip_patch = out_patch
pred = np.stack(smoothed, axis=0)
torch.cuda.empty_cache()
for p, f, xf, c in zip(pred, frames, f_frames, coords):
y1, y2, x1, x2 = c
p = cv2.resize(p.astype(np.uint8), (x2 - x1, y2 - y1))
ff = xf.copy()
ff[y1:y2, x1:x2] = p
# month region enhancement by GFPGAN
cropped_faces, restored_faces, restored_img = restorer.enhance(
ff, has_aligned=False, only_center_face=True, paste_back=True)
# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
mm = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 0, 0, 0]
mouse_mask = np.zeros_like(restored_img)
tmp_mask = enhancer.faceparser.process(restored_img[y1:y2, x1:x2], mm)[0]
tmp_mask = mask_postprocess(tmp_mask, thres=getattr(args, 'mask_feather', 20))
mouse_mask[y1:y2, x1:x2]= cv2.resize(tmp_mask, (x2 - x1, y2 - y1))[:, :, np.newaxis] / 255.
height, width = ff.shape[:2]
# Ensure shapes align for blending at native resolution
restored_img = restored_img if restored_img.shape[:2] == (height, width) else cv2.resize(restored_img, (width, height))
ff = ff if ff.shape[:2] == (height, width) else cv2.resize(ff, (width, height))
full_mask = np.float32(mouse_mask)
if full_mask.shape[:2] != (height, width):
full_mask = cv2.resize(full_mask, (width, height))
# --- Shape alignment fix for Laplacian blending ---
if restored_img.shape != ff.shape:
ff = cv2.resize(ff, (restored_img.shape[1], restored_img.shape[0]))
print(f"[Fix] Resized ff to {ff.shape} to match restored_img")
if full_mask.shape[:2] != restored_img.shape[:2]:
full_mask = cv2.resize(full_mask, (restored_img.shape[1], restored_img.shape[0]))
print(f"[Fix] Resized full_mask to {full_mask.shape} to match restored_img")
print("Shapes before blending:", restored_img.shape, ff.shape, full_mask.shape)
img = Laplacian_Pyramid_Blending_with_mask(restored_img, ff, full_mask[:, :, 0], 12)
img = np.clip(img, 0, 255)
if img.shape[:2] != (height, width):
pp = np.uint8(cv2.resize(img, (width, height)))
else:
pp = np.uint8(img)
pp, orig_faces, enhanced_faces = enhancer.process(pp, xf, bbox=c, face_enhance=True, possion_blending=False)
out.write(pp)
out.release()
if not os.path.isdir(os.path.dirname(args.outfile)):
os.makedirs(os.path.dirname(args.outfile), exist_ok=True)
if platform.system() == 'Windows':
command = ['ffmpeg', '-loglevel', 'error', '-y', '-i', args.audio, '-i', 'temp/{}/result.mp4'.format(args.tmp_dir), '-shortest', '-c:v', 'copy', args.outfile]
subprocess.call(command, shell=False)
else:
command = 'ffmpeg -loglevel error -y -i {} -i {} -shortest -c:v copy {}'.format(args.audio, 'temp/{}/result.mp4'.format(args.tmp_dir), args.outfile)
subprocess.call(command, shell=True)
print('outfile:', args.outfile)
# frames:256x256, full_frames: original size
def datagen(frames, mels, full_frames, frames_pil, cox):
img_batch, mel_batch, frame_batch, coords_batch, ref_batch, full_frame_batch = [], [], [], [], [], []
base_name = args.face.split('/')[-1]
refs = []
image_size = 256
# original frames
kp_extractor = KeypointExtractor()
fr_pil = [Image.fromarray(frame) for frame in frames]
lms = kp_extractor.extract_keypoint(fr_pil, 'temp/'+base_name+'x12_landmarks.txt')
frames_pil = [ (lm, frame) for frame,lm in zip(fr_pil, lms)] # frames is the croped version of modified face
crops, orig_images, quads = crop_faces(image_size, frames_pil, scale=1.0, use_fa=True)
# Guard against invalid (NaN/inf) quads by reusing last valid or falling back to identity
clean_quads = []
identity_quad = np.array([[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]], dtype=np.float32)
for q in quads:
if q is None or (not np.isfinite(q).all()):
q = clean_quads[-1] if len(clean_quads) > 0 else identity_quad
clean_quads.append(q)
# EMA smooth the alignment quads to reduce temporal jitter in warping
beta_q = float(getattr(args, 'quad_ema_beta', 0.0))
if beta_q > 0.0 and len(clean_quads) > 1:
sm_quads = []
prev_q = clean_quads[0].astype(np.float32)
sm_quads.append(prev_q)
for q in clean_quads[1:]:
qf = q.astype(np.float32)
prev_q = beta_q * prev_q + (1.0 - beta_q) * qf
sm_quads.append(prev_q)
else:
sm_quads = clean_quads
inverse_transforms = [calc_alignment_coefficients(q + 0.5, [[0, 0], [0, image_size], [image_size, image_size], [image_size, 0]]) for q in sm_quads]
del kp_extractor.detector
oy1,oy2,ox1,ox2 = cox
face_det_results = face_detect(full_frames, args, jaw_correction=True)
for inverse_transform, crop, full_frame, face_det in zip(inverse_transforms, crops, full_frames, face_det_results):
imc_pil = paste_image(inverse_transform, crop, Image.fromarray(
cv2.resize(full_frame[int(oy1):int(oy2), int(ox1):int(ox2)], (256, 256))))
ff = full_frame.copy()
ff[int(oy1):int(oy2), int(ox1):int(ox2)] = cv2.resize(np.array(imc_pil.convert('RGB')), (ox2 - ox1, oy2 - oy1))
oface, coords = face_det
y1, y2, x1, x2 = coords
refs.append(ff[y1: y2, x1:x2])
for i, m in enumerate(mels):
idx = 0 if args.static else i
frame_to_save = frames[idx].copy()
face = refs[idx]
oface, coords = face_det_results[idx].copy()
face = cv2.resize(face, (args.img_size, args.img_size))
oface = cv2.resize(oface, (args.img_size, args.img_size))
img_batch.append(oface)
ref_batch.append(face)
mel_batch.append(m)
coords_batch.append(coords)
frame_batch.append(frame_to_save)
full_frame_batch.append(full_frames[idx].copy())
if len(img_batch) >= args.LNet_batch_size:
img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch)
img_masked = img_batch.copy()
img_original = img_batch.copy()
img_masked[:, args.img_size//2:] = 0
img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255.
mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])
yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch
img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch, ref_batch = [], [], [], [], [], [], []
if len(img_batch) > 0:
img_batch, mel_batch, ref_batch = np.asarray(img_batch), np.asarray(mel_batch), np.asarray(ref_batch)
img_masked = img_batch.copy()
img_original = img_batch.copy()
img_masked[:, args.img_size//2:] = 0
img_batch = np.concatenate((img_masked, ref_batch), axis=3) / 255.
mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])
yield img_batch, mel_batch, frame_batch, coords_batch, img_original, full_frame_batch
if __name__ == '__main__':
main()