shunliwang

update

8bc3305 6 months ago

14.5 kB

'''
Create face mask and face boundary mask according to face landmarks,
so as to supervize the activation of Conv layer.
'''

import os
import numpy as np
import cv2
import dlib
import random
import argparse
from tqdm import tqdm
import time
from skimage import transform as trans
# from color_transfer import color_transfer
from .warp import gen_warp_params, warp_by_params, warp_mask


def crop_img_bbox(img, bbox, res, scale=1.3):
    x, y, w, h = bbox
    left, right = x, x+w
    top, bottom = y, y+h

    H, W, C = img.shape
    cx, cy = (left+right)//2, (top+bottom)//2
    w, h = (right-left)//2, (bottom-top)//2

    x1 = max(0, int(cx-w*scale))
    x2 = min(W, int(cx+w*scale))
    y1 = max(0, int(cy-h*scale))
    y2 = min(H, int(cy+h*scale))

    roi = img[y1:y2, x1:x2]
    roi = cv2.resize(roi, (res, res))

    return roi


def get_mask_center(mask):
    l, t, w, h = cv2.boundingRect(mask[:, :, 0:1].astype(np.uint8))
    center = int(l+w/2), int(t+h/2)
    return center


def get_5_keypoint(shape):
    def get_point(idx):
        # return [shape.part(idx).x, shape.part(idx).y]
        return shape[idx]

    def center(pt1, pt2):
        return [(pt1[0]+pt2[0])//2, (pt1[1]+pt2[1])//2]

    leye = np.array(center(get_point(36), get_point(39)),
                    dtype=int).reshape(-1, 2)
    reye = np.array(center(get_point(45), get_point(42)),
                    dtype=int).reshape(-1, 2)
    nose = np.array(get_point(30), dtype=int).reshape(-1, 2)
    lmouth = np.array(get_point(48),
                      dtype=int).reshape(-1, 2)
    rmouth = np.array(get_point(54),
                      dtype=int).reshape(-1, 2)

    pts = np.concatenate([leye, reye, nose, lmouth, rmouth], axis=0)

    return pts


def get_boundary(mask):
    if len(mask.shape) == 3:
        mask = mask[:, :, 0]
    mask = cv2.GaussianBlur(mask, (3, 3), 0)
    boundary = mask / 255.
    boundary = 4*boundary*(1.-boundary)
    return boundary


# def get_boundary(mask):
#     if len(mask.shape) == 3:
#         mask = mask[:, :, 0]
#     mask = cv2.GaussianBlur(mask, (3, 3), 0)
#     mask = mask.astype(np.uint8)

#     # Dilation and Erosion to find the boundary
#     dilated = cv2.dilate(mask, None, iterations=1)
#     boundary = cv2.subtract(dilated, mask)

#     # normalize the boundary to have values between 0 and 1
#     boundary = boundary / 255.

#     return boundary



def blur_mask(mask):
    blur_k = 2*np.random.randint(1, 10)-1
    
    #kernel = np.ones((blur_k+1, blur_k+1), np.uint8)
    #mask = cv2.erode(mask, kernel)
    
    mask = cv2.GaussianBlur(mask, (blur_k, blur_k), 0)


    return mask


def random_deform(pt, tgt, scale=0.3):
    x1, y1 = pt
    x2, y2 = tgt

    x = x1+(x2-x1)*np.random.rand()*scale
    y = y1+(y2-y1)*np.random.rand()*scale
    #print('before:', pt, ' after:', [int(x), int(y)])
    return [int(x), int(y)]


def get_specific_mask(img, shape, mtype='mouth', random_side=False):
    if mtype == 'eyes':
        landmarks = shape[42:45] if random.choice([True, False]) else shape[36:39]

    elif mtype == 'nose':
        landmarks = shape[27:35]

    elif mtype == 'mouth':
        landmarks = shape[48:60]

    elif mtype == 'eyebrows':
        landmarks = shape[22:26] if random.choice([True, False]) else shape[17:21]

    else:
        raise ValueError(f"Invalid mtype. Choose from 'eyes', 'nose', 'mouth', or 'eyebrows', but got {mtype}")

    # find convex hull
    hull = cv2.convexHull(landmarks)
    hull = hull.astype(int)

    # mask
    hull_mask = np.zeros_like(img)
    cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
    mask = hull_mask
    return mask


def get_hull_mask(img, shape, mtype='hull'):
    if mtype == 'normal-hull':
        landmarks = np.array(shape)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
        mask = hull_mask

    elif mtype == 'inner-hull':
        landmarks = shape[17:]
        landmarks = np.array(landmarks)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        mask = hull_mask

    elif mtype == 'inner-hull-no-eyebrow':
        landmarks = shape[27:]
        landmarks = np.array(landmarks)
        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        mask = hull_mask

    elif mtype == 'mouth-hull':
        landmarks = shape[2:15]
        #landmarks.append(shape[29])
        landmarks = np.concatenate([landmarks, shape[29].reshape(1, -1)], axis=0)

        # find convex hull
        hull = cv2.convexHull(landmarks)
        hull = hull.astype(int)

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        # kernel = np.ones((2, 2), np.uint8)
        # c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
        mask = hull_mask

    elif mtype == 'whole-hull':
        face_height = shape[9][1] - shape[22][1]
        landmarks = []
        for i in range(27):
            lmk = shape[i]
            if i >= 5 and i <= 11:
                x, y = lmk[0], lmk[1]
                lmk = [x, max(0, y+15)]
            # lift the eyebrows to get a larger landmark convex hull
            if i >= 18 and i <= 27:
                x, y = lmk[0], lmk[1]
                lmk = [x, max(0, y-face_height//4)]

            landmarks.append(lmk)

        # find convex hull
        landmarks = np.array(landmarks, dtype=np.int32)
        hull = cv2.convexHull(landmarks)
        hull = np.reshape(hull, (1, -1, 2))

        # full face mask
        hull_mask = np.zeros_like(img)
        cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

        # kernel = np.ones((2, 2), np.uint8)
        # c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
        mask = hull_mask
    '''
    elif mtype == 'rect':
        cnt = []
        for idx in [5, 11, 17, 26]:
            cnt.append(shape[idx])
        x, y, w, h = cv2.boundingRect(np.array(cnt))
        rect_mask = np.zeros_like(img)
        cv2.rectangle(rect_mask, (x, y), (x+w, y+h),
                      (255, 255, 255), cv2.FILLED)
        mask = rect_mask
    '''
    return mask


def get_mask(shape, img, std=20, deform=True, restrict_mask=None):
    mask_type = [
        'normal-hull', 
        'inner-hull', 
        'inner-hull-no-eyebrow', 
        'mouth-hull', 
        'whole-hull'
    ]
    max_mask = get_hull_mask(img, shape, 'whole-hull')
    if deform:
        mtype = mask_type[np.random.randint(len(mask_type))]
        if mtype == 'rect':
            mask = get_hull_mask(img, shape, 'inner-hull-no-eyebrow')
            x, y, w, h = cv2.boundingRect(mask[:,:,0])
            for i in range(y, y+h):
                for j in range(x, x+w):
                    for k in range(mask.shape[2]):
                        mask[i, j, k] = 255
        else:
            mask = get_hull_mask(img, shape, mtype)

        # random deform
        if np.random.rand() < 0.9:
            mask = warp_mask(mask, std=std)

        # # random erode/dilate
        # prob = np.random.rand()
        # if prob < 0.3:
        #     erode_k = 2*np.random.randint(1, 10)+1
        #     kernel = np.ones((erode_k, erode_k), np.uint8)
        #     mask = cv2.erode(mask, kernel)
        # elif prob < 0.6:
        #     erode_k = 2*np.random.randint(1, 10)+1
        #     kernel = np.ones((erode_k, erode_k), np.uint8)
        #     mask = cv2.dilate(mask, kernel)
    else:
        mask = max_mask.copy()
    
    if restrict_mask is not None:
        mask = mask*(restrict_mask//255)

    # restrict mask range
    mask = mask *(max_mask//255)

    # random blur
    if deform and np.random.rand() < 0.9:
        mask = blur_mask(mask)

    return mask[:,:,0]

def mask_postprocess(mask):
    # random erode/dilate
    prob = np.random.rand()
    if prob < 0.3:
        erode_k = 2*np.random.randint(1, 10)+1
        kernel = np.ones((erode_k, erode_k), np.uint8)
        mask = cv2.erode(mask, kernel)
    elif prob < 0.6:
        erode_k = 2*np.random.randint(1, 10)+1
        kernel = np.ones((erode_k, erode_k), np.uint8)
        mask = cv2.dilate(mask, kernel)
    
    # random blur
    if np.random.rand() < 0.9:
        mask = blur_mask(mask)

    return mask


def get_affine_param(from_, to_):
        # use skimage tranformation
    tform = trans.SimilarityTransform()
    tform.estimate(from_.astype(np.float32), to_.astype(
        np.float32))  # tform.estimate(from_, to_)
    M = tform.params[0:2, :]

    return M


def random_sharpen_img(img):
    cand = ['bsharpen', 'gsharpen']  # , 'none']
    mode = cand[np.random.randint(len(cand))]
    # print('sharpen mode:', mode)
    if mode == "bsharpen":
        # Sharpening using filter2D
        kernel = np.ones((3, 3)) * (-1)
        kernel[1, 1] = 9
        #kernel /= 9.
        out = cv2.filter2D(img, -1, kernel)
    elif mode == "gsharpen":
        # Sharpening using Weighted Method
        gaussain_blur = cv2.GaussianBlur(img, (0, 0), 3.0)
        out = cv2.addWeighted(
            img, 1.5, gaussain_blur, -0.5, 0, img)
    else:
        out = img

    return out


def random_blur_img(img):
    cand = ['avg', 'gaussion', 'med']  # , 'none']
    mode = cand[np.random.randint(len(cand))]
    # print('blur mode:', mode)
    ksize = 2*np.random.randint(1, 5)+1

    if mode == 'avg':
        # Averaging
        out = cv2.blur(img, (ksize, ksize))
    elif mode == 'gaussion':
        # Gaussian Blurring
        out = cv2.GaussianBlur(img, (ksize, ksize), 0)
    elif mode == 'med':
        # Median blurring
        out = cv2.medianBlur(img, ksize)
    else:
        out = img
    # elif mode == 'bilateral'
    #     # Bilateral Filtering
    #     out = cv2.bilateralFilter(img,9,75,75)

    return out


def random_warp_img(img, prob=0.5):
    H, W, C = img.shape
    param = gen_warp_params(W, flip=False)
    choice = [True, False]

    out = warp_by_params(param, img,
                         can_flip=False,  # choice[np.random.randint(2)],
                         can_transform=False,  # choice[np.random.randint(2)],
                         can_warp=(np.random.randint(10) < int(prob*10)),
                         border_replicate=choice[np.random.randint(2)])
    return out


def main(args):
    np.random.seed(int(time.time()))
    detector = dlib.get_frontal_face_detector()
    landmark_predictor = dlib.shape_predictor(args.model)

    src_im = cv2.imread(args.src)
    tgt_im = cv2.imread(args.tgt)

    H, W, C = tgt_im.shape
    src_im = cv2.resize(src_im, (W, H))

    def get_shape(img):
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        dets = detector(img, 1)
        det = dets[0]
        shape = landmark_predictor(img, det)

        return shape, det

    src_shape, src_det = get_shape(src_im)
    src_5_pts = get_5_keypoint(src_shape)
    src_mask = get_mask(src_shape, src_im, whole=True, deform=False)

    tgt_shape, tgt_det = get_shape(tgt_im)
    tgt_5_pts = get_5_keypoint(tgt_shape)
    tgt_mask = get_mask(tgt_shape, tgt_im, whole=False, deform=True)

    #aff_param = get_affine_param(src_5_pts, tgt_5_pts)

    # color transfer:
    mask = src_mask[:, :, 0:1]/255.
    ct_modes = ['lct', 'rct', 'idt', 'idt-m', 'mkl', 'mkl-m',
                'sot', 'sot-m', 'mix-m']  # , 'seamless-hist-match']
    for mode in ct_modes:
        colored_src = color_transfer(mode, src_im, tgt_im, mask)
        cv2.imwrite('{}_colored.png'.format(mode), colored_src)
    src_im = colored_src

    w1, h1 = src_det.right()-src_det.left(), src_det.bottom()-src_det.top()
    w2, h2 = tgt_det.right()-tgt_det.left(), tgt_det.bottom()-tgt_det.top()
    w_scale, h_scale = w2/w1, h2/h1

    scaled_src = cv2.resize(src_im, (int(W*w_scale), int(H*h_scale)))
    scaled_mask = cv2.resize(src_mask, (int(W*w_scale), int(H*h_scale)))

    src_5_pts[:, 0] = src_5_pts[:, 0]*w_scale
    src_5_pts[:, 1] = src_5_pts[:, 1]*h_scale
    aff_param = get_affine_param(src_5_pts, tgt_5_pts)

    aligned_src = cv2.warpAffine(
        scaled_src, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
    aligned_mask = cv2.warpAffine(
        scaled_mask, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)

    center = get_mask_center(aligned_mask)
    print('mask center:', center)
    # colored_src = transfer_color(aligned_src, tgt_im)

    init_blend = cv2.seamlessClone(
        aligned_src, tgt_im, aligned_mask, center, cv2.NORMAL_CLONE)
    cv2.imwrite('init_blended.png', init_blend)

    # aligned_blend = cv2.warpAffine(
    #    colored_blend, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
    b_mask = tgt_mask[:, :, 0:1]/255.
    out_blend = init_blend*b_mask + tgt_im*(1. - b_mask)
    cv2.imwrite('out_blended.png', out_blend)

    res = 256
    blend_crop = crop_img_bbox(out_blend, tgt_det, res, scale=1.5)
    mask_crop = crop_img_bbox(tgt_mask, tgt_det, res, scale=1.5)
    boundary = get_boundary(mask_crop)

    cv2.imwrite('crop_blend.png', blend_crop)
    cv2.imwrite('crop_mask.png', mask_crop)
    cv2.imwrite('crop_bound.png', boundary*255)


if __name__ == "__main__":
    p = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    p.add_argument('-s', '--src', type=str,
                   help='src image')
    p.add_argument('-t', '--tgt', type=str,
                   help='tgt image')
    p.add_argument('--model', type=str, default='/data1/yuchen/download/face_landmark/shape_predictor_68_face_landmarks.dat',
                   help="path to downloaded detector")
    args = p.parse_args()
    print(args)

    main(args)

	'''
	Create face mask and face boundary mask according to face landmarks,
	so as to supervize the activation of Conv layer.
	'''

	import os
	import numpy as np
	import cv2
	import dlib
	import random
	import argparse
	from tqdm import tqdm
	import time
	from skimage import transform as trans
	# from color_transfer import color_transfer
	from .warp import gen_warp_params, warp_by_params, warp_mask


	def crop_img_bbox(img, bbox, res, scale=1.3):
	x, y, w, h = bbox
	left, right = x, x+w
	top, bottom = y, y+h

	H, W, C = img.shape
	cx, cy = (left+right)//2, (top+bottom)//2
	w, h = (right-left)//2, (bottom-top)//2

	x1 = max(0, int(cx-w*scale))
	x2 = min(W, int(cx+w*scale))
	y1 = max(0, int(cy-h*scale))
	y2 = min(H, int(cy+h*scale))

	roi = img[y1:y2, x1:x2]
	roi = cv2.resize(roi, (res, res))

	return roi


	def get_mask_center(mask):
	l, t, w, h = cv2.boundingRect(mask[:, :, 0:1].astype(np.uint8))
	center = int(l+w/2), int(t+h/2)
	return center


	def get_5_keypoint(shape):
	def get_point(idx):
	# return [shape.part(idx).x, shape.part(idx).y]
	return shape[idx]

	def center(pt1, pt2):
	return [(pt1[0]+pt2[0])//2, (pt1[1]+pt2[1])//2]

	leye = np.array(center(get_point(36), get_point(39)),
	dtype=int).reshape(-1, 2)
	reye = np.array(center(get_point(45), get_point(42)),
	dtype=int).reshape(-1, 2)
	nose = np.array(get_point(30), dtype=int).reshape(-1, 2)
	lmouth = np.array(get_point(48),
	dtype=int).reshape(-1, 2)
	rmouth = np.array(get_point(54),
	dtype=int).reshape(-1, 2)

	pts = np.concatenate([leye, reye, nose, lmouth, rmouth], axis=0)

	return pts


	def get_boundary(mask):
	if len(mask.shape) == 3:
	mask = mask[:, :, 0]
	mask = cv2.GaussianBlur(mask, (3, 3), 0)
	boundary = mask / 255.
	boundary = 4boundary(1.-boundary)
	return boundary


	# def get_boundary(mask):
	# if len(mask.shape) == 3:
	# mask = mask[:, :, 0]
	# mask = cv2.GaussianBlur(mask, (3, 3), 0)
	# mask = mask.astype(np.uint8)

	# # Dilation and Erosion to find the boundary
	# dilated = cv2.dilate(mask, None, iterations=1)
	# boundary = cv2.subtract(dilated, mask)

	# # normalize the boundary to have values between 0 and 1
	# boundary = boundary / 255.

	# return boundary



	def blur_mask(mask):
	blur_k = 2*np.random.randint(1, 10)-1

	#kernel = np.ones((blur_k+1, blur_k+1), np.uint8)
	#mask = cv2.erode(mask, kernel)

	mask = cv2.GaussianBlur(mask, (blur_k, blur_k), 0)


	return mask


	def random_deform(pt, tgt, scale=0.3):
	x1, y1 = pt
	x2, y2 = tgt

	x = x1+(x2-x1)np.random.rand()scale
	y = y1+(y2-y1)np.random.rand()scale
	#print('before:', pt, ' after:', [int(x), int(y)])
	return [int(x), int(y)]


	def get_specific_mask(img, shape, mtype='mouth', random_side=False):
	if mtype == 'eyes':
	landmarks = shape[42:45] if random.choice([True, False]) else shape[36:39]

	elif mtype == 'nose':
	landmarks = shape[27:35]

	elif mtype == 'mouth':
	landmarks = shape[48:60]

	elif mtype == 'eyebrows':
	landmarks = shape[22:26] if random.choice([True, False]) else shape[17:21]

	else:
	raise ValueError(f"Invalid mtype. Choose from 'eyes', 'nose', 'mouth', or 'eyebrows', but got {mtype}")

	# find convex hull
	hull = cv2.convexHull(landmarks)
	hull = hull.astype(int)

	# mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
	mask = hull_mask
	return mask


	def get_hull_mask(img, shape, mtype='hull'):
	if mtype == 'normal-hull':
	landmarks = np.array(shape)

	# find convex hull
	hull = cv2.convexHull(landmarks)
	hull = hull.astype(int)

	# full face mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))
	mask = hull_mask

	elif mtype == 'inner-hull':
	landmarks = shape[17:]
	landmarks = np.array(landmarks)

	# find convex hull
	hull = cv2.convexHull(landmarks)
	hull = hull.astype(int)

	# full face mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

	mask = hull_mask

	elif mtype == 'inner-hull-no-eyebrow':
	landmarks = shape[27:]
	landmarks = np.array(landmarks)
	# find convex hull
	hull = cv2.convexHull(landmarks)
	hull = hull.astype(int)

	# full face mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

	mask = hull_mask

	elif mtype == 'mouth-hull':
	landmarks = shape[2:15]
	#landmarks.append(shape[29])
	landmarks = np.concatenate([landmarks, shape[29].reshape(1, -1)], axis=0)

	# find convex hull
	hull = cv2.convexHull(landmarks)
	hull = hull.astype(int)

	# full face mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

	# kernel = np.ones((2, 2), np.uint8)
	# c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
	mask = hull_mask

	elif mtype == 'whole-hull':
	face_height = shape[9][1] - shape[22][1]
	landmarks = []
	for i in range(27):
	lmk = shape[i]
	if i >= 5 and i <= 11:
	x, y = lmk[0], lmk[1]
	lmk = [x, max(0, y+15)]
	# lift the eyebrows to get a larger landmark convex hull
	if i >= 18 and i <= 27:
	x, y = lmk[0], lmk[1]
	lmk = [x, max(0, y-face_height//4)]

	landmarks.append(lmk)

	# find convex hull
	landmarks = np.array(landmarks, dtype=np.int32)
	hull = cv2.convexHull(landmarks)
	hull = np.reshape(hull, (1, -1, 2))

	# full face mask
	hull_mask = np.zeros_like(img)
	cv2.fillPoly(hull_mask, [hull], (255, 255, 255))

	# kernel = np.ones((2, 2), np.uint8)
	# c_mask = cv2.dilate(hull_mask, kernel, iterations=1)
	mask = hull_mask
	'''
	elif mtype == 'rect':
	cnt = []
	for idx in [5, 11, 17, 26]:
	cnt.append(shape[idx])
	x, y, w, h = cv2.boundingRect(np.array(cnt))
	rect_mask = np.zeros_like(img)
	cv2.rectangle(rect_mask, (x, y), (x+w, y+h),
	(255, 255, 255), cv2.FILLED)
	mask = rect_mask
	'''
	return mask


	def get_mask(shape, img, std=20, deform=True, restrict_mask=None):
	mask_type = [
	'normal-hull',
	'inner-hull',
	'inner-hull-no-eyebrow',
	'mouth-hull',
	'whole-hull'
	]
	max_mask = get_hull_mask(img, shape, 'whole-hull')
	if deform:
	mtype = mask_type[np.random.randint(len(mask_type))]
	if mtype == 'rect':
	mask = get_hull_mask(img, shape, 'inner-hull-no-eyebrow')
	x, y, w, h = cv2.boundingRect(mask[:,:,0])
	for i in range(y, y+h):
	for j in range(x, x+w):
	for k in range(mask.shape[2]):
	mask[i, j, k] = 255
	else:
	mask = get_hull_mask(img, shape, mtype)

	# random deform
	if np.random.rand() < 0.9:
	mask = warp_mask(mask, std=std)

	# # random erode/dilate
	# prob = np.random.rand()
	# if prob < 0.3:
	# erode_k = 2*np.random.randint(1, 10)+1
	# kernel = np.ones((erode_k, erode_k), np.uint8)
	# mask = cv2.erode(mask, kernel)
	# elif prob < 0.6:
	# erode_k = 2*np.random.randint(1, 10)+1
	# kernel = np.ones((erode_k, erode_k), np.uint8)
	# mask = cv2.dilate(mask, kernel)
	else:
	mask = max_mask.copy()

	if restrict_mask is not None:
	mask = mask*(restrict_mask//255)

	# restrict mask range
	mask = mask *(max_mask//255)

	# random blur
	if deform and np.random.rand() < 0.9:
	mask = blur_mask(mask)

	return mask[:,:,0]

	def mask_postprocess(mask):
	# random erode/dilate
	prob = np.random.rand()
	if prob < 0.3:
	erode_k = 2*np.random.randint(1, 10)+1
	kernel = np.ones((erode_k, erode_k), np.uint8)
	mask = cv2.erode(mask, kernel)
	elif prob < 0.6:
	erode_k = 2*np.random.randint(1, 10)+1
	kernel = np.ones((erode_k, erode_k), np.uint8)
	mask = cv2.dilate(mask, kernel)

	# random blur
	if np.random.rand() < 0.9:
	mask = blur_mask(mask)

	return mask


	def get_affine_param(from_, to_):
	# use skimage tranformation
	tform = trans.SimilarityTransform()
	tform.estimate(from_.astype(np.float32), to_.astype(
	np.float32)) # tform.estimate(from_, to_)
	M = tform.params[0:2, :]

	return M


	def random_sharpen_img(img):
	cand = ['bsharpen', 'gsharpen'] # , 'none']
	mode = cand[np.random.randint(len(cand))]
	# print('sharpen mode:', mode)
	if mode == "bsharpen":
	# Sharpening using filter2D
	kernel = np.ones((3, 3)) * (-1)
	kernel[1, 1] = 9
	#kernel /= 9.
	out = cv2.filter2D(img, -1, kernel)
	elif mode == "gsharpen":
	# Sharpening using Weighted Method
	gaussain_blur = cv2.GaussianBlur(img, (0, 0), 3.0)
	out = cv2.addWeighted(
	img, 1.5, gaussain_blur, -0.5, 0, img)
	else:
	out = img

	return out


	def random_blur_img(img):
	cand = ['avg', 'gaussion', 'med'] # , 'none']
	mode = cand[np.random.randint(len(cand))]
	# print('blur mode:', mode)
	ksize = 2*np.random.randint(1, 5)+1

	if mode == 'avg':
	# Averaging
	out = cv2.blur(img, (ksize, ksize))
	elif mode == 'gaussion':
	# Gaussian Blurring
	out = cv2.GaussianBlur(img, (ksize, ksize), 0)
	elif mode == 'med':
	# Median blurring
	out = cv2.medianBlur(img, ksize)
	else:
	out = img
	# elif mode == 'bilateral'
	# # Bilateral Filtering
	# out = cv2.bilateralFilter(img,9,75,75)

	return out


	def random_warp_img(img, prob=0.5):
	H, W, C = img.shape
	param = gen_warp_params(W, flip=False)
	choice = [True, False]

	out = warp_by_params(param, img,
	can_flip=False, # choice[np.random.randint(2)],
	can_transform=False, # choice[np.random.randint(2)],
	can_warp=(np.random.randint(10) < int(prob*10)),
	border_replicate=choice[np.random.randint(2)])
	return out


	def main(args):
	np.random.seed(int(time.time()))
	detector = dlib.get_frontal_face_detector()
	landmark_predictor = dlib.shape_predictor(args.model)

	src_im = cv2.imread(args.src)
	tgt_im = cv2.imread(args.tgt)

	H, W, C = tgt_im.shape
	src_im = cv2.resize(src_im, (W, H))

	def get_shape(img):
	img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
	dets = detector(img, 1)
	det = dets[0]
	shape = landmark_predictor(img, det)

	return shape, det

	src_shape, src_det = get_shape(src_im)
	src_5_pts = get_5_keypoint(src_shape)
	src_mask = get_mask(src_shape, src_im, whole=True, deform=False)

	tgt_shape, tgt_det = get_shape(tgt_im)
	tgt_5_pts = get_5_keypoint(tgt_shape)
	tgt_mask = get_mask(tgt_shape, tgt_im, whole=False, deform=True)

	#aff_param = get_affine_param(src_5_pts, tgt_5_pts)

	# color transfer:
	mask = src_mask[:, :, 0:1]/255.
	ct_modes = ['lct', 'rct', 'idt', 'idt-m', 'mkl', 'mkl-m',
	'sot', 'sot-m', 'mix-m'] # , 'seamless-hist-match']
	for mode in ct_modes:
	colored_src = color_transfer(mode, src_im, tgt_im, mask)
	cv2.imwrite('{}_colored.png'.format(mode), colored_src)
	src_im = colored_src

	w1, h1 = src_det.right()-src_det.left(), src_det.bottom()-src_det.top()
	w2, h2 = tgt_det.right()-tgt_det.left(), tgt_det.bottom()-tgt_det.top()
	w_scale, h_scale = w2/w1, h2/h1

	scaled_src = cv2.resize(src_im, (int(Ww_scale), int(Hh_scale)))
	scaled_mask = cv2.resize(src_mask, (int(Ww_scale), int(Hh_scale)))

	src_5_pts[:, 0] = src_5_pts[:, 0]*w_scale
	src_5_pts[:, 1] = src_5_pts[:, 1]*h_scale
	aff_param = get_affine_param(src_5_pts, tgt_5_pts)

	aligned_src = cv2.warpAffine(
	scaled_src, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
	aligned_mask = cv2.warpAffine(
	scaled_mask, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)

	center = get_mask_center(aligned_mask)
	print('mask center:', center)
	# colored_src = transfer_color(aligned_src, tgt_im)

	init_blend = cv2.seamlessClone(
	aligned_src, tgt_im, aligned_mask, center, cv2.NORMAL_CLONE)
	cv2.imwrite('init_blended.png', init_blend)

	# aligned_blend = cv2.warpAffine(
	# colored_blend, aff_param, (W, H), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REFLECT)
	b_mask = tgt_mask[:, :, 0:1]/255.
	out_blend = init_blendb_mask + tgt_im(1. - b_mask)
	cv2.imwrite('out_blended.png', out_blend)

	res = 256
	blend_crop = crop_img_bbox(out_blend, tgt_det, res, scale=1.5)
	mask_crop = crop_img_bbox(tgt_mask, tgt_det, res, scale=1.5)
	boundary = get_boundary(mask_crop)

	cv2.imwrite('crop_blend.png', blend_crop)
	cv2.imwrite('crop_mask.png', mask_crop)
	cv2.imwrite('crop_bound.png', boundary*255)


	if __name__ == "__main__":
	p = argparse.ArgumentParser(
	formatter_class=argparse.ArgumentDefaultsHelpFormatter
	)
	p.add_argument('-s', '--src', type=str,
	help='src image')
	p.add_argument('-t', '--tgt', type=str,
	help='tgt image')
	p.add_argument('--model', type=str, default='/data1/yuchen/download/face_landmark/shape_predictor_68_face_landmarks.dat',
	help="path to downloaded detector")
	args = p.parse_args()
	print(args)

	main(args)