Download utils/inference_utils.py from Anirudh1950/VideoReTalk: direct link, hf CLI and curl.
- Browser
- Download file 24 kB
-
https://huggingface.co/Anirudh1950/VideoReTalk/resolve/main/utils/inference_utils.py
- Command line
-
hf download hf://Anirudh1950/VideoReTalk/utils/inference_utils.py
-
curl -L -o inference_utils.py https://huggingface.co/Anirudh1950/VideoReTalk/resolve/main/utils/inference_utils.py
24 kB
| import numpy as np | |
| import cv2, argparse, torch | |
| import torchvision.transforms.functional as TF | |
| from models import load_network, load_DNet | |
| from tqdm import tqdm | |
| from PIL import Image | |
| from scipy.spatial import ConvexHull | |
| from third_part import face_detection | |
| from third_part.face3d.models import networks | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| def options(): | |
| parser = argparse.ArgumentParser(description='Inference code to lip-sync videos in the wild using Wav2Lip models') | |
| parser.add_argument('--DNet_path', type=str, default='checkpoints/DNet.pt') | |
| parser.add_argument('--LNet_path', type=str, default='checkpoints/LNet.pth') | |
| parser.add_argument('--ENet_path', type=str, default='checkpoints/ENet.pth') | |
| parser.add_argument('--face3d_net_path', type=str, default='checkpoints/face3d_pretrain_epoch_20.pth') | |
| parser.add_argument('--face', type=str, help='Filepath of video/image that contains faces to use', required=True) | |
| parser.add_argument('--audio', type=str, help='Filepath of video/audio file to use as raw audio source', required=True) | |
| parser.add_argument('--exp_img', type=str, help='Expression template. neutral, smile or image path', default='neutral') | |
| parser.add_argument('--outfile', type=str, help='Video path to save result') | |
| parser.add_argument('--fps', type=float, help='Can be specified only if input is a static image (default: 25)', default=25., required=False) | |
| parser.add_argument('--pads', nargs='+', type=int, default=[0, 30, 0, 0], help='Padding (top, bottom, left, right). Please adjust to include chin at least') | |
| parser.add_argument('--face_det_batch_size', type=int, help='Batch size for face detection', default=8) | |
| parser.add_argument('--LNet_batch_size', type=int, help='Batch size for LNet', default=32) | |
| parser.add_argument('--img_size', type=int, default=512) | |
| parser.add_argument('--mel_step_size', type=int, default=12, | |
| help='Number of mel frames per chunk. Lower = tighter sync, higher = smoother') | |
| parser.add_argument('--mel_multiplier_scale', type=float, default=1.0, | |
| help='Scale factor for mel-to-video frame mapping. <1 if lips ahead; >1 if lagging') | |
| parser.add_argument('--gpen_size', type=int, default=512, | |
| help='GPEN enhancer internal size (e.g., 512 or 1024)') | |
| parser.add_argument('--gpen_model', type=str, default='GPEN-BFR-512', | |
| help='GPEN model name to load from checkpoints (e.g., GPEN-BFR-512)') | |
| # Audio preprocessing controls | |
| parser.add_argument('--audio_preprocess', action='store_true', default=True, | |
| help='Preprocess audio with ffmpeg: force 16kHz mono and apply optional filters') | |
| parser.add_argument('--audio_normalize', action='store_true', default=True, | |
| help='Normalize audio levels (ffmpeg dynaudnorm) during preprocessing') | |
| parser.add_argument('--audio_denoise', action='store_true', default=False, | |
| help='Apply simple spectral denoise (ffmpeg afftdn) during preprocessing') | |
| parser.add_argument('--audio_target_sr', type=int, default=16000, | |
| help='Target audio sampling rate for preprocessing') | |
| parser.add_argument('--audio_eps_on_nan', action='store_true', default=True, | |
| help='If mel contains NaN, add epsilon noise to audio and retry once') | |
| parser.add_argument('--crop', nargs='+', type=int, default=[0, -1, 0, -1], | |
| help='Crop video to a smaller region (top, bottom, left, right). Applied after resize_factor and rotate arg. ' | |
| 'Useful if multiple face present. -1 implies the value will be auto-inferred based on height, width') | |
| parser.add_argument('--box', nargs='+', type=int, default=[-1, -1, -1, -1], | |
| help='Specify a constant bounding box for the face. Use only as a last resort if the face is not detected.' | |
| 'Also, might work only if the face is not moving around much. Syntax: (top, bottom, left, right).') | |
| parser.add_argument('--nosmooth', default=False, action='store_true', help='Prevent smoothing face detections over a short temporal window') | |
| parser.add_argument('--static', default=False, action='store_true') | |
| parser.add_argument('--up_face', default='original') | |
| parser.add_argument('--one_shot', action='store_true') | |
| parser.add_argument('--without_rl1', default=False, action='store_true', help='Do not use the relative l1') | |
| parser.add_argument('--tmp_dir', type=str, default='temp', help='Folder to save tmp results') | |
| parser.add_argument('--re_preprocess', action='store_true') | |
| parser.add_argument('--temporal_smooth_window', type=int, default=5, | |
| help='Window size for smoothing face detections over time when temporal smoothing is enabled') | |
| parser.add_argument('--lip_temporal_smooth', action='store_true', | |
| help='Enable temporal smoothing for synthesized lip frames before blending back to the video') | |
| parser.add_argument('--temporal_blend_weight', type=float, default=0.3, | |
| help='Blend weight used when lip temporal smoothing is enabled (0 disables, closer to 1 increases smoothing)') | |
| parser.add_argument('--temporal_blend_auto', action='store_true', default=False, | |
| help='Adapt smoothing by mel energy (less smoothing on loud phonemes)') | |
| parser.add_argument('--temporal_blend_k', type=float, default=5.0, | |
| help='Sharpness of energy-to-weight mapping when temporal_blend_auto is enabled') | |
| parser.add_argument('--mouth_region_smoothing', type=float, default=None, | |
| help='Override lip temporal smoothing weight (0..1). Higher reduces jitter, lower keeps more detail') | |
| parser.add_argument('--lip_temporal_mode', type=str, default='ema', choices=['ema', 'median', 'both'], | |
| help='Temporal smoothing mode for lips: ema (prev-frame), median (window), or both') | |
| parser.add_argument('--lip_temporal_window', type=int, default=5, | |
| help='Window size for median temporal smoothing when enabled') | |
| parser.add_argument('--mel_offset_frames', type=int, default=0, | |
| help='Global offset in mel frames applied to alignment (positive -> later)') | |
| parser.add_argument('--auto_sync_calibrate', action='store_true', default=True, | |
| help='Estimate best mel-frame offset by correlating mouth openness and mel energy') | |
| parser.add_argument('--calibrate_window', type=int, default=6, | |
| help='Max absolute frame offset to search during calibration') | |
| parser.add_argument('--auto_sync_scale', action='store_true', default=False, | |
| help='Also search for a mel/frame scale factor during sync calibration') | |
| parser.add_argument('--sync_scale_range', type=float, default=0.08, | |
| help='Percent range (+/-) for scale search when auto_sync_scale is enabled (e.g. 0.08 => ±8%)') | |
| parser.add_argument('--sync_scale_steps', type=int, default=5, | |
| help='Number of steps on each side of scale search (total samples = 2*steps+1)') | |
| # Sync gating controls | |
| parser.add_argument('--enable_syncnet_gate', action='store_true', default=False, | |
| help='Gate mouth blending based on lip–audio sync confidence') | |
| parser.add_argument('--lip_sync_conf_threshold', type=float, default=0.85, | |
| help='Confidence threshold (0..1) to accept mouth update when gating is enabled') | |
| parser.add_argument('--syncnet_window', type=int, default=5, | |
| help='Temporal window (frames) for sync confidence correlation') | |
| parser.add_argument('--syncnet_model_path', type=str, default='checkpoints/syncnet.pt', | |
| help='Optional TorchScript SyncNet model path; proxy confidence used if missing') | |
| parser.add_argument('--lock_box_from_first', action='store_true', | |
| help='Detect box on first frame and reuse for all frames to reduce jitter') | |
| parser.add_argument('--mask_feather', type=int, default=20, | |
| help='Feather width (in pixels) for mouth mask postprocess to reduce seams') | |
| parser.add_argument('--bbox_ema_beta', type=float, default=0.6, | |
| help='EMA smoothing factor for face detection boxes (0 disables)') | |
| parser.add_argument('--quad_ema_beta', type=float, default=0.5, | |
| help='EMA smoothing factor for alignment quads (used in warping)') | |
| # Chunking and warmup controls | |
| parser.add_argument('--enable_chunking', action='store_true', default=True, | |
| help='Automatically split long videos into chunks for stable sync') | |
| parser.add_argument('--chunk_duration', type=float, default=5.0, | |
| help='Chunk duration in seconds (set 0 to disable)') | |
| parser.add_argument('--chunk_overlap', type=float, default=0.1, | |
| help='Overlap in seconds between consecutive chunks') | |
| parser.add_argument('--chunk_crossfade', action='store_true', default=False, | |
| help='Apply crossfade between chunk boundaries (re-encodes)') | |
| parser.add_argument('--chunk_temp_dir', type=str, default='parts', | |
| help='Subfolder under temp/<tmp_dir> to store chunk files') | |
| parser.add_argument('--warmup_frames', type=int, default=2, | |
| help='Run warmup frames per chunk before writing outputs') | |
| # Optical flow temporal smoothing controls | |
| parser.add_argument('--flow_temporal_smooth', action='store_true', | |
| help='Enable optical-flow-guided temporal smoothing of the lip patch') | |
| parser.add_argument('--flow_blend_weight', type=float, default=0.4, | |
| help='Blend weight for warped previous lip patch (0..1)') | |
| parser.add_argument('--flow_pyr_scale', type=float, default=0.5, | |
| help='Farneback pyramid scale (0..1)') | |
| parser.add_argument('--flow_levels', type=int, default=3, | |
| help='Farneback number of pyramid levels') | |
| parser.add_argument('--flow_winsize', type=int, default=21, | |
| help='Farneback averaging window size (odd)') | |
| parser.add_argument('--flow_iters', type=int, default=3, | |
| help='Farneback iterations per pyramid level') | |
| parser.add_argument('--flow_poly_n', type=int, default=5, | |
| help='Farneback size of pixel neighborhood (odd)') | |
| parser.add_argument('--flow_poly_sigma', type=float, default=1.2, | |
| help='Farneback standard deviation of Gaussian used') | |
| parser.add_argument('--latent_noise_scale', type=float, default=0.0, | |
| help='Scale of latent noise injected into mel features (set 0 to disable)') | |
| # Optional full-frame stabilization using ffmpeg vidstab | |
| parser.add_argument('--pre_stabilize_video', action='store_true', default=False, | |
| help='Apply ffmpeg vidstab stabilization pass on the input video before processing') | |
| parser.add_argument('--stabilize_shakiness', type=int, default=5, | |
| help='vidstabdetect shakiness parameter (1-10). Higher handles larger shakes but is slower') | |
| parser.add_argument('--stabilize_smoothing', type=int, default=30, | |
| help='vidstabtransform smoothing parameter controlling strength of stabilization') | |
| parser.add_argument('--stabilize_crop', type=str, default='keep', | |
| help='vidstabtransform cropping behaviour: keep, content, or black') | |
| # (SyncNet gating args defined above to avoid duplicates) | |
| args = parser.parse_args() | |
| return args | |
| exp_aus_dict = { # AU01_r, AU02_r, AU04_r, AU05_r, AU06_r, AU07_r, AU09_r, AU10_r, AU12_r, AU14_r, AU15_r, AU17_r, AU20_r, AU23_r, AU25_r, AU26_r, AU45_r. | |
| 'sad': torch.Tensor([[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), | |
| 'angry':torch.Tensor([[0, 0, 0.3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]), | |
| 'surprise': torch.Tensor([[0, 0, 0, 0.2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]) | |
| } | |
| def mask_postprocess(mask, thres=20): | |
| # Clamp feather threshold to mask size | |
| h, w = mask.shape[:2] | |
| t = int(max(1, min(thres, min(h, w) // 4))) | |
| mask[:t, :] = 0; mask[-t:, :] = 0 | |
| mask[:, :t] = 0; mask[:, -t:] = 0 | |
| # Choose a valid odd Gaussian kernel <= min(h, w) | |
| base = max(3, min(101, (min(h, w) // 2) * 2 + 1)) | |
| if base % 2 == 0: | |
| base = max(3, base - 1) | |
| sigma = max(1.0, base / 9.0) | |
| mask = cv2.GaussianBlur(mask, (base, base), sigma) | |
| mask = cv2.GaussianBlur(mask, (base, base), sigma) | |
| return mask.astype(np.float32) | |
| def trans_image(image): | |
| image = TF.resize( | |
| image, size=256, interpolation=Image.BICUBIC) | |
| image = TF.to_tensor(image) | |
| image = TF.normalize(image, mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) | |
| return image | |
| def obtain_seq_index(index, num_frames): | |
| seq = list(range(index-13, index+13)) | |
| seq = [ min(max(item, 0), num_frames-1) for item in seq ] | |
| return seq | |
| def transform_semantic(semantic, frame_index, crop_norm_ratio=None): | |
| index = obtain_seq_index(frame_index, semantic.shape[0]) | |
| coeff_3dmm = semantic[index,...] | |
| ex_coeff = coeff_3dmm[:,80:144] #expression # 64 | |
| angles = coeff_3dmm[:,224:227] #euler angles for pose | |
| translation = coeff_3dmm[:,254:257] #translation | |
| crop = coeff_3dmm[:,259:262] #crop param | |
| if crop_norm_ratio: | |
| crop[:, -3] = crop[:, -3] * crop_norm_ratio | |
| coeff_3dmm = np.concatenate([ex_coeff, angles, translation, crop], 1) | |
| return torch.Tensor(coeff_3dmm).permute(1,0) | |
| def find_crop_norm_ratio(source_coeff, target_coeffs): | |
| alpha = 0.3 | |
| exp_diff = np.mean(np.abs(target_coeffs[:,80:144] - source_coeff[:,80:144]), 1) # mean different exp | |
| angle_diff = np.mean(np.abs(target_coeffs[:,224:227] - source_coeff[:,224:227]), 1) # mean different angle | |
| index = np.argmin(alpha*exp_diff + (1-alpha)*angle_diff) # find the smallerest index | |
| crop_norm_ratio = source_coeff[:,-3] / target_coeffs[index:index+1, -3] | |
| return crop_norm_ratio | |
| def get_smoothened_boxes(boxes, T): | |
| for i in range(len(boxes)): | |
| if i + T > len(boxes): | |
| window = boxes[len(boxes) - T:] | |
| else: | |
| window = boxes[i : i + T] | |
| boxes[i] = np.mean(window, axis=0) | |
| return boxes | |
| def face_detect(images, args, jaw_correction=False, detector=None): | |
| if detector == None: | |
| device = 'cuda:0' if torch.cuda.is_available() else 'cpu' | |
| detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D, | |
| flip_input=False, device=device) | |
| # If user provided a fixed bounding box, respect it for all frames | |
| if hasattr(args, 'box') and args.box is not None and len(args.box) == 4 and all([v != -1 for v in args.box]): | |
| top, bottom, left, right = args.box | |
| pady1, pady2, padx1, padx2 = args.pads if jaw_correction else (0,30,0,0) | |
| results = [] | |
| for image in images: | |
| h, w = image.shape[:2] | |
| y1 = max(0, top - pady1) | |
| y2 = min(h, bottom + pady2) | |
| x1 = max(0, left - padx1) | |
| x2 = min(w, right + padx2) | |
| results.append([x1, y1, x2, y2]) | |
| boxes = np.array(results) | |
| results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)] | |
| del detector | |
| torch.cuda.empty_cache() | |
| return results | |
| # Optionally lock the detection box from the first frame for all subsequent frames | |
| if getattr(args, 'lock_box_from_first', False): | |
| pady1, pady2, padx1, padx2 = args.pads if jaw_correction else (0,30,0,0) | |
| first = images[0] | |
| rect0 = detector.get_detections_for_batch(np.array([first]))[0] | |
| if rect0 is None: | |
| h, w = first.shape[:2] | |
| rect0 = [0, 0, w, h] | |
| y1 = max(0, rect0[1] - pady1) | |
| y2 = min(first.shape[0], rect0[3] + pady2) | |
| x1 = max(0, rect0[0] - padx1) | |
| x2 = min(first.shape[1], rect0[2] + padx2) | |
| boxes = np.array([[x1, y1, x2, y2] for _ in images]) | |
| results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image in images] | |
| del detector | |
| torch.cuda.empty_cache() | |
| return results | |
| batch_size = args.face_det_batch_size | |
| while 1: | |
| predictions = [] | |
| try: | |
| for i in tqdm(range(0, len(images), batch_size),desc='FaceDet:'): | |
| predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size]))) | |
| except RuntimeError: | |
| if batch_size == 1: | |
| raise RuntimeError('Image too big to run face detection on GPU. Please use the --resize_factor argument') | |
| batch_size //= 2 | |
| print('Recovering from OOM error; New batch size: {}'.format(batch_size)) | |
| continue | |
| break | |
| results = [] | |
| pady1, pady2, padx1, padx2 = args.pads if jaw_correction else (0,30,0,0) | |
| for rect, image in zip(predictions, images): | |
| if rect is None: | |
| # Fallback to full frame if detection fails, and continue | |
| h, w = image.shape[:2] | |
| rect = [0, 0, w, h] | |
| y1 = max(0, rect[1] - pady1) | |
| y2 = min(image.shape[0], rect[3] + pady2) | |
| x1 = max(0, rect[0] - padx1) | |
| x2 = min(image.shape[1], rect[2] + padx2) | |
| results.append([x1, y1, x2, y2]) | |
| boxes = np.array(results) | |
| if not args.nosmooth: | |
| smooth_window = max(1, args.temporal_smooth_window) | |
| boxes = get_smoothened_boxes(boxes, T=smooth_window) | |
| # Apply EMA smoothing across frames for additional stability | |
| beta = float(getattr(args, 'bbox_ema_beta', 0.0)) | |
| if beta > 0.0 and len(boxes) > 1: | |
| ema_boxes = [boxes[0].astype(np.float32)] | |
| prev = ema_boxes[0] | |
| for b in boxes[1:]: | |
| b = b.astype(np.float32) | |
| prev = beta * prev + (1.0 - beta) * b | |
| ema_boxes.append(prev) | |
| boxes = np.stack(ema_boxes, axis=0) | |
| # Convert to safe integer boxes per-frame and clamp to image bounds | |
| int_boxes = [] | |
| for image, (x1, y1, x2, y2) in zip(images, boxes): | |
| h, w = image.shape[:2] | |
| x1i, y1i, x2i, y2i = [int(round(v)) for v in (x1, y1, x2, y2)] | |
| x1i = max(0, min(x1i, w - 1)) | |
| x2i = max(0, min(x2i, w)) | |
| y1i = max(0, min(y1i, h - 1)) | |
| y2i = max(0, min(y2i, h)) | |
| # Ensure valid ordering and non-empty area | |
| if x2i <= x1i: | |
| if x1i + 1 < w: | |
| x2i = x1i + 1 | |
| else: | |
| x1i = max(0, w - 2) | |
| x2i = w - 1 | |
| if y2i <= y1i: | |
| if y1i + 1 < h: | |
| y2i = y1i + 1 | |
| else: | |
| y1i = max(0, h - 2) | |
| y2i = h - 1 | |
| int_boxes.append((x1i, y1i, x2i, y2i)) | |
| results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, int_boxes)] | |
| del detector | |
| torch.cuda.empty_cache() | |
| return results | |
| def _load(checkpoint_path, device): | |
| if device == 'cuda': | |
| checkpoint = torch.load(checkpoint_path) | |
| else: | |
| checkpoint = torch.load(checkpoint_path, | |
| map_location=lambda storage, loc: storage) | |
| return checkpoint | |
| def split_coeff(coeffs): | |
| """ | |
| Return: | |
| coeffs_dict -- a dict of torch.tensors | |
| Parameters: | |
| coeffs -- torch.tensor, size (B, 256) | |
| """ | |
| id_coeffs = coeffs[:, :80] | |
| exp_coeffs = coeffs[:, 80: 144] | |
| tex_coeffs = coeffs[:, 144: 224] | |
| angles = coeffs[:, 224: 227] | |
| gammas = coeffs[:, 227: 254] | |
| translations = coeffs[:, 254:] | |
| return { | |
| 'id': id_coeffs, | |
| 'exp': exp_coeffs, | |
| 'tex': tex_coeffs, | |
| 'angle': angles, | |
| 'gamma': gammas, | |
| 'trans': translations | |
| } | |
| def Laplacian_Pyramid_Blending_with_mask(A, B, m, num_levels = 6): | |
| # generate Gaussian pyramid for A,B and mask | |
| GA = A.copy() | |
| GB = B.copy() | |
| GM = m.copy() | |
| gpA = [GA] | |
| gpB = [GB] | |
| gpM = [GM] | |
| for i in range(num_levels): | |
| GA = cv2.pyrDown(GA) | |
| GB = cv2.pyrDown(GB) | |
| GM = cv2.pyrDown(GM) | |
| gpA.append(np.float32(GA)) | |
| gpB.append(np.float32(GB)) | |
| gpM.append(np.float32(GM)) | |
| # generate Laplacian Pyramids for A,B and masks | |
| lpA = [gpA[num_levels-1]] # the bottom of the Lap-pyr holds the last (smallest) Gauss level | |
| lpB = [gpB[num_levels-1]] | |
| gpMr = [gpM[num_levels-1]] | |
| for i in range(num_levels-1,0,-1): | |
| # Laplacian: subtract upscaled version of lower level from current level | |
| # to get the high frequencies | |
| LA = np.subtract(gpA[i-1], cv2.pyrUp(gpA[i])) | |
| LB = np.subtract(gpB[i-1], cv2.pyrUp(gpB[i])) | |
| lpA.append(LA) | |
| lpB.append(LB) | |
| gpMr.append(gpM[i-1]) # also reverse the masks | |
| # Now blend images according to mask in each level | |
| LS = [] | |
| for la,lb,gm in zip(lpA,lpB,gpMr): | |
| gm = gm[:,:,np.newaxis] | |
| ls = la * gm + lb * (1.0 - gm) | |
| LS.append(ls) | |
| # now reconstruct | |
| ls_ = LS[0] | |
| for i in range(1,num_levels): | |
| ls_ = cv2.pyrUp(ls_) | |
| # --- Fix: handle shape mismatches between pyramid levels --- | |
| if ls_.shape[:2] != LS[i].shape[:2]: | |
| LS[i] = cv2.resize(LS[i], (ls_.shape[1], ls_.shape[0])) | |
| print(f"[Fix] Resized LS[{i}] from pyramid to match {ls_.shape}") | |
| ls_ = cv2.add(ls_, LS[i]) | |
| return ls_ | |
| def load_model(args, device): | |
| D_Net = load_DNet(args).to(device) | |
| model = load_network(args).to(device) | |
| return D_Net, model | |
| def normalize_kp(kp_source, kp_driving, kp_driving_initial, adapt_movement_scale=False, | |
| use_relative_movement=False, use_relative_jacobian=False): | |
| if adapt_movement_scale: | |
| source_area = ConvexHull(kp_source['value'][0].data.cpu().numpy()).volume | |
| driving_area = ConvexHull(kp_driving_initial['value'][0].data.cpu().numpy()).volume | |
| adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area) | |
| else: | |
| adapt_movement_scale = 1 | |
| kp_new = {k: v for k, v in kp_driving.items()} | |
| if use_relative_movement: | |
| kp_value_diff = (kp_driving['value'] - kp_driving_initial['value']) | |
| kp_value_diff *= adapt_movement_scale | |
| kp_new['value'] = kp_value_diff + kp_source['value'] | |
| if use_relative_jacobian: | |
| jacobian_diff = torch.matmul(kp_driving['jacobian'], torch.inverse(kp_driving_initial['jacobian'])) | |
| kp_new['jacobian'] = torch.matmul(jacobian_diff, kp_source['jacobian']) | |
| return kp_new | |
| def load_face3d_net(ckpt_path, device): | |
| net_recon = networks.define_net_recon(net_recon='resnet50', use_last_fc=False, init_path='').to(device) | |
| checkpoint = torch.load(ckpt_path, map_location=device) | |
| net_recon.load_state_dict(checkpoint['net_recon']) | |
| net_recon.eval() | |
| return net_recon | |