import colorsys import math from enum import IntEnum import cv2 import numpy as np import numpy.linalg as npla from core import imagelib from core import mathlib from facelib import FaceType from core.mathlib.umeyama import umeyama landmarks_2D = np.array([ [ 0.000213256, 0.106454 ], #17 [ 0.0752622, 0.038915 ], #18 [ 0.18113, 0.0187482 ], #19 [ 0.29077, 0.0344891 ], #20 [ 0.393397, 0.0773906 ], #21 [ 0.586856, 0.0773906 ], #22 [ 0.689483, 0.0344891 ], #23 [ 0.799124, 0.0187482 ], #24 [ 0.904991, 0.038915 ], #25 [ 0.98004, 0.106454 ], #26 [ 0.490127, 0.203352 ], #27 [ 0.490127, 0.307009 ], #28 [ 0.490127, 0.409805 ], #29 [ 0.490127, 0.515625 ], #30 [ 0.36688, 0.587326 ], #31 [ 0.426036, 0.609345 ], #32 [ 0.490127, 0.628106 ], #33 [ 0.554217, 0.609345 ], #34 [ 0.613373, 0.587326 ], #35 [ 0.121737, 0.216423 ], #36 [ 0.187122, 0.178758 ], #37 [ 0.265825, 0.179852 ], #38 [ 0.334606, 0.231733 ], #39 [ 0.260918, 0.245099 ], #40 [ 0.182743, 0.244077 ], #41 [ 0.645647, 0.231733 ], #42 [ 0.714428, 0.179852 ], #43 [ 0.793132, 0.178758 ], #44 [ 0.858516, 0.216423 ], #45 [ 0.79751, 0.244077 ], #46 [ 0.719335, 0.245099 ], #47 [ 0.254149, 0.780233 ], #48 [ 0.340985, 0.745405 ], #49 [ 0.428858, 0.727388 ], #50 [ 0.490127, 0.742578 ], #51 [ 0.551395, 0.727388 ], #52 [ 0.639268, 0.745405 ], #53 [ 0.726104, 0.780233 ], #54 [ 0.642159, 0.864805 ], #55 [ 0.556721, 0.902192 ], #56 [ 0.490127, 0.909281 ], #57 [ 0.423532, 0.902192 ], #58 [ 0.338094, 0.864805 ], #59 [ 0.290379, 0.784792 ], #60 [ 0.428096, 0.778746 ], #61 [ 0.490127, 0.785343 ], #62 [ 0.552157, 0.778746 ], #63 [ 0.689874, 0.784792 ], #64 [ 0.553364, 0.824182 ], #65 [ 0.490127, 0.831803 ], #66 [ 0.42689 , 0.824182 ] #67 ], dtype=np.float32) landmarks_2D_new = np.array([ [ 0.000213256, 0.106454 ], #17 [ 0.0752622, 0.038915 ], #18 [ 0.18113, 0.0187482 ], #19 [ 0.29077, 0.0344891 ], #20 [ 0.393397, 0.0773906 ], #21 [ 0.586856, 0.0773906 ], #22 [ 0.689483, 0.0344891 ], #23 [ 0.799124, 0.0187482 ], #24 [ 0.904991, 0.038915 ], #25 [ 0.98004, 0.106454 ], #26 [ 0.490127, 0.203352 ], #27 [ 0.490127, 0.307009 ], #28 [ 0.490127, 0.409805 ], #29 [ 0.490127, 0.515625 ], #30 [ 0.36688, 0.587326 ], #31 [ 0.426036, 0.609345 ], #32 [ 0.490127, 0.628106 ], #33 [ 0.554217, 0.609345 ], #34 [ 0.613373, 0.587326 ], #35 [ 0.121737, 0.216423 ], #36 [ 0.187122, 0.178758 ], #37 [ 0.265825, 0.179852 ], #38 [ 0.334606, 0.231733 ], #39 [ 0.260918, 0.245099 ], #40 [ 0.182743, 0.244077 ], #41 [ 0.645647, 0.231733 ], #42 [ 0.714428, 0.179852 ], #43 [ 0.793132, 0.178758 ], #44 [ 0.858516, 0.216423 ], #45 [ 0.79751, 0.244077 ], #46 [ 0.719335, 0.245099 ], #47 [ 0.254149, 0.780233 ], #48 [ 0.726104, 0.780233 ], #54 ], dtype=np.float32) mouth_center_landmarks_2D = np.array([ [-4.4202591e-07, 4.4916576e-01], #48 [ 1.8399176e-01, 3.7537053e-01], #49 [ 3.7018123e-01, 3.3719531e-01], #50 [ 5.0000089e-01, 3.6938059e-01], #51 [ 6.2981832e-01, 3.3719531e-01], #52 [ 8.1600773e-01, 3.7537053e-01], #53 [ 1.0000000e+00, 4.4916576e-01], #54 [ 8.2213330e-01, 6.2836081e-01], #55 [ 6.4110327e-01, 7.0757812e-01], #56 [ 5.0000089e-01, 7.2259867e-01], #57 [ 3.5889623e-01, 7.0757812e-01], #58 [ 1.7786618e-01, 6.2836081e-01], #59 [ 7.6765373e-02, 4.5882553e-01], #60 [ 3.6856663e-01, 4.4601500e-01], #61 [ 5.0000089e-01, 4.5999300e-01], #62 [ 6.3143289e-01, 4.4601500e-01], #63 [ 9.2323411e-01, 4.5882553e-01], #64 [ 6.3399029e-01, 5.4228687e-01], #65 [ 5.0000089e-01, 5.5843467e-01], #66 [ 3.6601129e-01, 5.4228687e-01] #67 ], dtype=np.float32) # 68 point landmark definitions landmarks_68_pt = { "mouth": (48,68), "right_eyebrow": (17, 22), "left_eyebrow": (22, 27), "right_eye": (36, 42), "left_eye": (42, 48), "nose": (27, 36), # missed one point "jaw": (0, 17) } landmarks_68_3D = np.array( [ [-73.393523 , -29.801432 , 47.667532 ], #00 [-72.775014 , -10.949766 , 45.909403 ], #01 [-70.533638 , 7.929818 , 44.842580 ], #02 [-66.850058 , 26.074280 , 43.141114 ], #03 [-59.790187 , 42.564390 , 38.635298 ], #04 [-48.368973 , 56.481080 , 30.750622 ], #05 [-34.121101 , 67.246992 , 18.456453 ], #06 [-17.875411 , 75.056892 , 3.609035 ], #07 [0.098749 , 77.061286 , -0.881698 ], #08 [17.477031 , 74.758448 , 5.181201 ], #09 [32.648966 , 66.929021 , 19.176563 ], #10 [46.372358 , 56.311389 , 30.770570 ], #11 [57.343480 , 42.419126 , 37.628629 ], #12 [64.388482 , 25.455880 , 40.886309 ], #13 [68.212038 , 6.990805 , 42.281449 ], #14 [70.486405 , -11.666193 , 44.142567 ], #15 [71.375822 , -30.365191 , 47.140426 ], #16 [-61.119406 , -49.361602 , 14.254422 ], #17 [-51.287588 , -58.769795 , 7.268147 ], #18 [-37.804800 , -61.996155 , 0.442051 ], #19 [-24.022754 , -61.033399 , -6.606501 ], #20 [-11.635713 , -56.686759 , -11.967398 ], #21 [12.056636 , -57.391033 , -12.051204 ], #22 [25.106256 , -61.902186 , -7.315098 ], #23 [38.338588 , -62.777713 , -1.022953 ], #24 [51.191007 , -59.302347 , 5.349435 ], #25 [60.053851 , -50.190255 , 11.615746 ], #26 [0.653940 , -42.193790 , -13.380835 ], #27 [0.804809 , -30.993721 , -21.150853 ], #28 [0.992204 , -19.944596 , -29.284036 ], #29 [1.226783 , -8.414541 , -36.948060 ], #00 [-14.772472 , 2.598255 , -20.132003 ], #01 [-7.180239 , 4.751589 , -23.536684 ], #02 [0.555920 , 6.562900 , -25.944448 ], #03 [8.272499 , 4.661005 , -23.695741 ], #04 [15.214351 , 2.643046 , -20.858157 ], #05 [-46.047290 , -37.471411 , 7.037989 ], #06 [-37.674688 , -42.730510 , 3.021217 ], #07 [-27.883856 , -42.711517 , 1.353629 ], #08 [-19.648268 , -36.754742 , -0.111088 ], #09 [-28.272965 , -35.134493 , -0.147273 ], #10 [-38.082418 , -34.919043 , 1.476612 ], #11 [19.265868 , -37.032306 , -0.665746 ], #12 [27.894191 , -43.342445 , 0.247660 ], #13 [37.437529 , -43.110822 , 1.696435 ], #14 [45.170805 , -38.086515 , 4.894163 ], #15 [38.196454 , -35.532024 , 0.282961 ], #16 [28.764989 , -35.484289 , -1.172675 ], #17 [-28.916267 , 28.612716 , -2.240310 ], #18 [-17.533194 , 22.172187 , -15.934335 ], #19 [-6.684590 , 19.029051 , -22.611355 ], #20 [0.381001 , 20.721118 , -23.748437 ], #21 [8.375443 , 19.035460 , -22.721995 ], #22 [18.876618 , 22.394109 , -15.610679 ], #23 [28.794412 , 28.079924 , -3.217393 ], #24 [19.057574 , 36.298248 , -14.987997 ], #25 [8.956375 , 39.634575 , -22.554245 ], #26 [0.381549 , 40.395647 , -23.591626 ], #27 [-7.428895 , 39.836405 , -22.406106 ], #28 [-18.160634 , 36.677899 , -15.121907 ], #29 [-24.377490 , 28.677771 , -4.785684 ], #30 [-6.897633 , 25.475976 , -20.893742 ], #31 [0.340663 , 26.014269 , -22.220479 ], #32 [8.444722 , 25.326198 , -21.025520 ], #33 [24.474473 , 28.323008 , -5.712776 ], #34 [8.449166 , 30.596216 , -20.671489 ], #35 [0.205322 , 31.408738 , -21.903670 ], #36 [-7.198266 , 30.844876 , -20.328022 ] #37 ], dtype=np.float32) FaceType_to_padding_remove_align = { FaceType.HALF: (0.0, False), FaceType.MID_FULL: (0.0675, False), FaceType.FULL: (0.2109375, False), FaceType.FULL_NO_ALIGN: (0.2109375, True), FaceType.WHOLE_FACE: (0.40, False), FaceType.HEAD: (0.70, False), FaceType.HEAD_NO_ALIGN: (0.70, True), } def convert_98_to_68(lmrks): #jaw result = [ lmrks[0] ] for i in range(2,16,2): result += [ ( lmrks[i] + (lmrks[i-1]+lmrks[i+1])/2 ) / 2 ] result += [ lmrks[16] ] for i in range(18,32,2): result += [ ( lmrks[i] + (lmrks[i-1]+lmrks[i+1])/2 ) / 2 ] result += [ lmrks[32] ] #eyebrows averaging result += [ lmrks[33], (lmrks[34]+lmrks[41])/2, (lmrks[35]+lmrks[40])/2, (lmrks[36]+lmrks[39])/2, (lmrks[37]+lmrks[38])/2, ] result += [ (lmrks[42]+lmrks[50])/2, (lmrks[43]+lmrks[49])/2, (lmrks[44]+lmrks[48])/2, (lmrks[45]+lmrks[47])/2, lmrks[46] ] #nose result += list ( lmrks[51:60] ) #left eye (from our view) result += [ lmrks[60], lmrks[61], lmrks[63], lmrks[64], lmrks[65], lmrks[67] ] #right eye result += [ lmrks[68], lmrks[69], lmrks[71], lmrks[72], lmrks[73], lmrks[75] ] #mouth result += list ( lmrks[76:96] ) return np.concatenate (result).reshape ( (68,2) ) def transform_points(points, mat, invert=False): if invert: mat = cv2.invertAffineTransform (mat) points = np.expand_dims(points, axis=1) points = cv2.transform(points, mat, points.shape) points = np.squeeze(points) return points def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] # get corner points in global space g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True) g_c = g_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (g_p[2]-g_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (g_p[1]-g_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) mod = (1.0 / scale)* ( npla.norm(g_p[0]-g_p[2])*(padding*np.sqrt(2.0) + 0.5) ) if face_type == FaceType.WHOLE_FACE: # adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead vec = (g_p[0]-g_p[3]).astype(np.float32) vec_len = npla.norm(vec) vec /= vec_len g_c += vec*vec_len*0.07 elif face_type == FaceType.HEAD: # assuming image_landmarks are 3D_Landmarks extracted for HEAD, # adjust horizontal offset according to estimated yaw yaw = estimate_averaged_yaw(transform_points (image_landmarks, mat, False)) hvec = (g_p[0]-g_p[1]).astype(np.float32) hvec_len = npla.norm(hvec) hvec /= hvec_len yaw *= np.abs(math.tanh(yaw*2)) # Damp near zero g_c -= hvec * (yaw * hvec_len / 2.0) # adjust vertical offset for HEAD, 50% below vvec = (g_p[0]-g_p[3]).astype(np.float32) vvec_len = npla.norm(vvec) vvec /= vvec_len g_c += vvec*vvec_len*0.50 # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array( [ g_c - tb_diag_vec*mod, g_c + bt_diag_vec*mod, g_c + tb_diag_vec*mod ] ) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array( [ g_c - tb_diag_vec*mod, g_c + bt_diag_vec*mod, g_c + tb_diag_vec*mod, g_c - bt_diag_vec*mod, ] ) # get area of face square in global space area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array( [ g_c + [-side,-side], g_c + [ side,-side], g_c + [ side, side] ] ) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) return mat def get_rect_from_landmarks(image_landmarks): mat = get_transform_mat(image_landmarks, 256, FaceType.FULL_NO_ALIGN) g_p = transform_points ( np.float32([(0,0),(255,255) ]) , mat, True) (l,t,r,b) = g_p[0][0], g_p[0][1], g_p[1][0], g_p[1][1] return (l,t,r,b) def expand_eyebrows(lmrks, eyebrows_expand_mod=1.0): if len(lmrks) != 68: raise Exception('works only with 68 landmarks') lmrks = np.array( lmrks.copy(), dtype=np.int ) # #nose ml_pnt = (lmrks[36] + lmrks[0]) // 2 mr_pnt = (lmrks[16] + lmrks[45]) // 2 # mid points between the mid points and eye ql_pnt = (lmrks[36] + ml_pnt) // 2 qr_pnt = (lmrks[45] + mr_pnt) // 2 # Top of the eye arrays bot_l = np.array((ql_pnt, lmrks[36], lmrks[37], lmrks[38], lmrks[39])) bot_r = np.array((lmrks[42], lmrks[43], lmrks[44], lmrks[45], qr_pnt)) # Eyebrow arrays top_l = lmrks[17:22] top_r = lmrks[22:27] # Adjust eyebrow arrays lmrks[17:22] = top_l + eyebrows_expand_mod * 0.5 * (top_l - bot_l) lmrks[22:27] = top_r + eyebrows_expand_mod * 0.5 * (top_r - bot_r) return lmrks def get_image_hull_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0 ): hull_mask = np.zeros(image_shape[0:2]+(1,),dtype=np.float32) lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) r_jaw = (lmrks[0:9], lmrks[17:18]) l_jaw = (lmrks[8:17], lmrks[26:27]) r_cheek = (lmrks[17:20], lmrks[8:9]) l_cheek = (lmrks[24:27], lmrks[8:9]) nose_ridge = (lmrks[19:25], lmrks[8:9],) r_eye = (lmrks[17:22], lmrks[27:28], lmrks[31:36], lmrks[8:9]) l_eye = (lmrks[22:27], lmrks[27:28], lmrks[31:36], lmrks[8:9]) nose = (lmrks[27:31], lmrks[31:36]) parts = [r_jaw, l_jaw, r_cheek, l_cheek, nose_ridge, r_eye, l_eye, nose] for item in parts: merged = np.concatenate(item) cv2.fillConvexPoly(hull_mask, cv2.convexHull(merged), (1,) ) return hull_mask def get_image_eye_mask (image_shape, image_landmarks): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') h,w,c = image_shape hull_mask = np.zeros( (h,w,1),dtype=np.float32) image_landmarks = image_landmarks.astype(np.int) cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[36:42]), (1,) ) cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[42:48]), (1,) ) dilate = h // 32 hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(dilate,dilate)), iterations = 1 ) blur = h // 16 blur = blur + (1-blur % 2) hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur) , 0) hull_mask = hull_mask[...,None] return hull_mask def get_image_mouth_mask (image_shape, image_landmarks): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') h,w,c = image_shape hull_mask = np.zeros( (h,w,1),dtype=np.float32) image_landmarks = image_landmarks.astype(np.int) cv2.fillConvexPoly( hull_mask, cv2.convexHull( image_landmarks[60:]), (1,) ) dilate = h // 32 hull_mask = cv2.dilate(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(dilate,dilate)), iterations = 1 ) blur = h // 16 blur = blur + (1-blur % 2) hull_mask = cv2.GaussianBlur(hull_mask, (blur, blur) , 0) hull_mask = hull_mask[...,None] return hull_mask def alpha_to_color (img_alpha, color): if len(img_alpha.shape) == 2: img_alpha = img_alpha[...,None] h,w,c = img_alpha.shape result = np.zeros( (h,w, len(color) ), dtype=np.float32 ) result[:,:] = color return result * img_alpha def get_cmask (image_shape, lmrks, eyebrows_expand_mod=1.0): h,w,c = image_shape hull = get_image_hull_mask (image_shape, lmrks, eyebrows_expand_mod ) result = np.zeros( (h,w,3), dtype=np.float32 ) def process(w,h, data ): d = {} cur_lc = 0 all_lines = [] for s, pts_loop_ar in data: lines = [] for pts, loop in pts_loop_ar: pts_len = len(pts) lines.append ( [ [ pts[i], pts[(i+1) % pts_len ] ] for i in range(pts_len - (0 if loop else 1) ) ] ) lines = np.concatenate (lines) lc = lines.shape[0] all_lines.append(lines) d[s] = cur_lc, cur_lc+lc cur_lc += lc all_lines = np.concatenate (all_lines, 0) #calculate signed distance for all points and lines line_count = all_lines.shape[0] pts_count = w*h all_lines = np.repeat ( all_lines[None,...], pts_count, axis=0 ).reshape ( (pts_count*line_count,2,2) ) pts = np.empty( (h,w,line_count,2), dtype=np.float32 ) pts[...,1] = np.arange(h)[:,None,None] pts[...,0] = np.arange(w)[:,None] pts = pts.reshape ( (h*w*line_count, -1) ) a = all_lines[:,0,:] b = all_lines[:,1,:] pa = pts-a ba = b-a ph = np.clip ( np.einsum('ij,ij->i', pa, ba) / np.einsum('ij,ij->i', ba, ba), 0, 1 ) dists = npla.norm ( pa - ba*ph[...,None], axis=1).reshape ( (h,w,line_count) ) def get_dists(name, thickness=0): s,e = d[name] result = dists[...,s:e] if thickness != 0: result = np.abs(result)-thickness return np.min (result, axis=-1) return get_dists l_eye = lmrks[42:48] r_eye = lmrks[36:42] l_brow = lmrks[22:27] r_brow = lmrks[17:22] mouth = lmrks[48:60] up_nose = np.concatenate( (lmrks[27:31], lmrks[33:34]) ) down_nose = lmrks[31:36] nose = np.concatenate ( (up_nose, down_nose) ) gdf = process ( w,h, ( ('eyes', ((l_eye, True), (r_eye, True)) ), ('brows', ((l_brow, False), (r_brow,False)) ), ('up_nose', ((up_nose, False),) ), ('down_nose', ((down_nose, False),) ), ('mouth', ((mouth, True),) ), ) ) eyes_fall_dist = w // 32 eyes_thickness = max( w // 64, 1 ) brows_fall_dist = w // 32 brows_thickness = max( w // 256, 1 ) nose_fall_dist = w / 12 nose_thickness = max( w // 96, 1 ) mouth_fall_dist = w // 32 mouth_thickness = max( w // 64, 1 ) eyes_mask = gdf('eyes',eyes_thickness) eyes_mask = 1-np.clip( eyes_mask/ eyes_fall_dist, 0, 1) #eyes_mask = np.clip ( 1- ( np.sqrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1) #eyes_mask = np.clip ( 1- ( np.cbrt( np.maximum(eyes_mask,0) ) / eyes_fall_dist ), 0, 1) brows_mask = gdf('brows', brows_thickness) brows_mask = 1-np.clip( brows_mask / brows_fall_dist, 0, 1) #brows_mask = np.clip ( 1- ( np.sqrt( np.maximum(brows_mask,0) ) / brows_fall_dist ), 0, 1) mouth_mask = gdf('mouth', mouth_thickness) mouth_mask = 1-np.clip( mouth_mask / mouth_fall_dist, 0, 1) #mouth_mask = np.clip ( 1- ( np.sqrt( np.maximum(mouth_mask,0) ) / mouth_fall_dist ), 0, 1) def blend(a,b,k): x = np.clip ( 0.5+0.5*(b-a)/k, 0.0, 1.0 ) return (a-b)*x+b - k*x*(1.0-x) #nose_mask = (a-b)*x+b - k*x*(1.0-x) #nose_mask = np.minimum (up_nose_mask , down_nose_mask ) #nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1) nose_mask = blend ( gdf('up_nose', nose_thickness), gdf('down_nose', nose_thickness), nose_thickness*3 ) nose_mask = 1-np.clip( nose_mask / nose_fall_dist, 0, 1) up_nose_mask = gdf('up_nose', nose_thickness) up_nose_mask = 1-np.clip( up_nose_mask / nose_fall_dist, 0, 1) #up_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(up_nose_mask,0) ) / nose_fall_dist ), 0, 1) down_nose_mask = gdf('down_nose', nose_thickness) down_nose_mask = 1-np.clip( down_nose_mask / nose_fall_dist, 0, 1) #down_nose_mask = np.clip ( 1- ( np.cbrt( np.maximum(down_nose_mask,0) ) / nose_fall_dist ), 0, 1) #nose_mask = np.clip( up_nose_mask + down_nose_mask, 0, 1 ) #nose_mask /= np.max(nose_mask) #nose_mask = np.maximum (up_nose_mask , down_nose_mask ) #nose_mask = down_nose_mask #nose_mask = np.zeros_like(nose_mask) eyes_mask = eyes_mask * (1-mouth_mask) nose_mask = nose_mask * (1-eyes_mask) hull_mask = hull[...,0].copy() hull_mask = hull_mask * (1-eyes_mask) * (1-brows_mask) * (1-nose_mask) * (1-mouth_mask) #eyes_mask = eyes_mask * (1-nose_mask) mouth_mask= mouth_mask * (1-nose_mask) brows_mask = brows_mask * (1-nose_mask)* (1-eyes_mask ) hull_mask = alpha_to_color(hull_mask, (0,1,0) ) eyes_mask = alpha_to_color(eyes_mask, (1,0,0) ) brows_mask = alpha_to_color(brows_mask, (0,0,1) ) nose_mask = alpha_to_color(nose_mask, (0,1,1) ) mouth_mask = alpha_to_color(mouth_mask, (0,0,1) ) #nose_mask = np.maximum( up_nose_mask, down_nose_mask ) result = hull_mask + mouth_mask+ nose_mask + brows_mask + eyes_mask result *= hull #result = np.clip (result, 0, 1) return result def blur_image_hull_mask (hull_mask): maxregion = np.argwhere(hull_mask==1.0) miny,minx = maxregion.min(axis=0)[:2] maxy,maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx; leny = maxy - miny; masky = int(minx+(lenx//2)) maskx = int(miny+(leny//2)) lowest_len = min (lenx, leny) ero = int( lowest_len * 0.085 ) blur = int( lowest_len * 0.10 ) hull_mask = cv2.erode(hull_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 ) hull_mask = cv2.blur(hull_mask, (blur, blur) ) hull_mask = np.expand_dims (hull_mask,-1) return hull_mask mirror_idxs = [ [0,16], [1,15], [2,14], [3,13], [4,12], [5,11], [6,10], [7,9], [17,26], [18,25], [19,24], [20,23], [21,22], [36,45], [37,44], [38,43], [39,42], [40,47], [41,46], [31,35], [32,34], [50,52], [49,53], [48,54], [59,55], [58,56], [67,65], [60,64], [61,63] ] def mirror_landmarks (landmarks, val): result = landmarks.copy() for idx in mirror_idxs: result [ idx ] = result [ idx[::-1] ] result[:,0] = val - result[:,0] - 1 return result def get_face_struct_mask (image_shape, image_landmarks, eyebrows_expand_mod=1.0, color=(1,) ): mask = np.zeros(image_shape[0:2]+( len(color),),dtype=np.float32) lmrks = expand_eyebrows(image_landmarks, eyebrows_expand_mod) draw_landmarks (mask, image_landmarks, color=color, draw_circles=False, thickness=2) return mask def draw_landmarks (image, image_landmarks, color=(0,255,0), draw_circles=True, thickness=1, transparent_mask=False): if len(image_landmarks) != 68: raise Exception('get_image_eye_mask works only with 68 landmarks') int_lmrks = np.array(image_landmarks, dtype=np.int) jaw = int_lmrks[slice(*landmarks_68_pt["jaw"])] right_eyebrow = int_lmrks[slice(*landmarks_68_pt["right_eyebrow"])] left_eyebrow = int_lmrks[slice(*landmarks_68_pt["left_eyebrow"])] mouth = int_lmrks[slice(*landmarks_68_pt["mouth"])] right_eye = int_lmrks[slice(*landmarks_68_pt["right_eye"])] left_eye = int_lmrks[slice(*landmarks_68_pt["left_eye"])] nose = int_lmrks[slice(*landmarks_68_pt["nose"])] # open shapes cv2.polylines(image, tuple(np.array([v]) for v in ( right_eyebrow, jaw, left_eyebrow, np.concatenate((nose, [nose[-6]])) )), False, color, thickness=thickness, lineType=cv2.LINE_AA) # closed shapes cv2.polylines(image, tuple(np.array([v]) for v in (right_eye, left_eye, mouth)), True, color, thickness=thickness, lineType=cv2.LINE_AA) if draw_circles: # the rest of the cicles for x, y in np.concatenate((right_eyebrow, left_eyebrow, mouth, right_eye, left_eye, nose), axis=0): cv2.circle(image, (x, y), 1, color, 1, lineType=cv2.LINE_AA) # jaw big circles for x, y in jaw: cv2.circle(image, (x, y), 2, color, lineType=cv2.LINE_AA) if transparent_mask: mask = get_image_hull_mask (image.shape, image_landmarks) image[...] = ( image * (1-mask) + image * mask / 2 )[...] def draw_rect_landmarks (image, rect, image_landmarks, face_type, face_size=256, transparent_mask=False, landmarks_color=(0,255,0)): draw_landmarks(image, image_landmarks, color=landmarks_color, transparent_mask=transparent_mask) imagelib.draw_rect (image, rect, (255,0,0), 2 ) image_to_face_mat = get_transform_mat (image_landmarks, face_size, face_type) points = transform_points ( [ (0,0), (0,face_size-1), (face_size-1, face_size-1), (face_size-1,0) ], image_to_face_mat, True) imagelib.draw_polygon (image, points, (0,0,255), 2) points = transform_points ( [ ( int(face_size*0.05), 0), ( int(face_size*0.1), int(face_size*0.1) ), ( 0, int(face_size*0.1) ) ], image_to_face_mat, True) imagelib.draw_polygon (image, points, (0,0,255), 2) def calc_face_pitch(landmarks): if not isinstance(landmarks, np.ndarray): landmarks = np.array (landmarks) t = ( (landmarks[6][1]-landmarks[8][1]) + (landmarks[10][1]-landmarks[8][1]) ) / 2.0 b = landmarks[8][1] return float(b-t) def estimate_averaged_yaw(landmarks): # Works much better than solvePnP if landmarks from "3DFAN" if not isinstance(landmarks, np.ndarray): landmarks = np.array (landmarks) l = ( (landmarks[27][0]-landmarks[0][0]) + (landmarks[28][0]-landmarks[1][0]) + (landmarks[29][0]-landmarks[2][0]) ) / 3.0 r = ( (landmarks[16][0]-landmarks[27][0]) + (landmarks[15][0]-landmarks[28][0]) + (landmarks[14][0]-landmarks[29][0]) ) / 3.0 return float(r-l) def estimate_pitch_yaw_roll(aligned_landmarks, size=256): """ returns pitch,yaw,roll [-pi/2...+pi/2] """ shape = (size,size) focal_length = shape[1] camera_center = (shape[1] / 2, shape[0] / 2) camera_matrix = np.array( [[focal_length, 0, camera_center[0]], [0, focal_length, camera_center[1]], [0, 0, 1]], dtype=np.float32) (_, rotation_vector, _) = cv2.solvePnP( np.concatenate( (landmarks_68_3D[:27], landmarks_68_3D[30:36]) , axis=0) , np.concatenate( (aligned_landmarks[:27], aligned_landmarks[30:36]) , axis=0).astype(np.float32), camera_matrix, np.zeros((4, 1)) ) pitch, yaw, roll = mathlib.rotationMatrixToEulerAngles( cv2.Rodrigues(rotation_vector)[0] ) half_pi = math.pi / 2.0 pitch = np.clip ( pitch, -half_pi, half_pi ) yaw = np.clip ( yaw , -half_pi, half_pi ) roll = np.clip ( roll, -half_pi, half_pi ) return -pitch, yaw, roll #if remove_align: # bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True) # #import code # #code.interact(local=dict(globals(), **locals())) # area = mathlib.polygon_area(bbox[:,0], bbox[:,1] ) # side = math.sqrt(area) / 2 # center = transform_points ( [(output_size/2,output_size/2)], mat, True) # pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] )) # pts2 = np.float32([[0,0],[output_size,0],[0,output_size]]) # mat = cv2.getAffineTransform(pts1,pts2) #if full_face_align_top and (face_type == FaceType.FULL or face_type == FaceType.FULL_NO_ALIGN): # #lmrks2 = expand_eyebrows(image_landmarks) # #lmrks2_ = transform_points( [ lmrks2[19], lmrks2[24] ], mat, False ) # #y_diff = np.float32( (0,np.min(lmrks2_[:,1])) ) # #y_diff = transform_points( [ np.float32( (0,0) ), y_diff], mat, True) # #y_diff = y_diff[1]-y_diff[0] # # x_diff = np.float32((0,0)) # # lmrks2_ = transform_points( [ image_landmarks[0], image_landmarks[16] ], mat, False ) # if lmrks2_[0,0] < 0: # x_diff = lmrks2_[0,0] # x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True) # x_diff = x_diff[1]-x_diff[0] # elif lmrks2_[1,0] >= output_size: # x_diff = lmrks2_[1,0]-(output_size-1) # x_diff = transform_points( [ np.float32( (0,0) ), np.float32((x_diff,0)) ], mat, True) # x_diff = x_diff[1]-x_diff[0] # # mat = cv2.getAffineTransform( l_t+y_diff+x_diff ,pts2) """ def get_averaged_transform_mat (img_landmarks, img_landmarks_prev, img_landmarks_next, average_frame_count, average_center_frame_count, output_size, face_type, scale=1.0): l_c_list = [] tb_diag_vec_list = [] bt_diag_vec_list = [] mod_list = [] count = max(average_frame_count,average_center_frame_count) for i in range ( -count, count+1, 1 ): if i < 0: lmrks = img_landmarks_prev[i] if -i < len(img_landmarks_prev) else None elif i > 0: lmrks = img_landmarks_next[i] if i < len(img_landmarks_next) else None else: lmrks = img_landmarks if lmrks is None: continue l_c, tb_diag_vec, bt_diag_vec, mod = get_transform_mat_data (lmrks, face_type, scale=scale) if i >= -average_frame_count and i <= average_frame_count: tb_diag_vec_list.append(tb_diag_vec) bt_diag_vec_list.append(bt_diag_vec) mod_list.append(mod) if i >= -average_center_frame_count and i <= average_center_frame_count: l_c_list.append(l_c) tb_diag_vec = np.mean( np.array(tb_diag_vec_list), axis=0 ) bt_diag_vec = np.mean( np.array(bt_diag_vec_list), axis=0 ) mod = np.mean( np.array(mod_list), axis=0 ) l_c = np.mean( np.array(l_c_list), axis=0 ) return get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type) def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) # get face padding value for FaceType padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] # get corner points in global space l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True) l_c = l_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) ) # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ) ] ) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ), np.round( l_c - bt_diag_vec*mod ), ] ) # get area of face square in global space area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array( [ np.round( l_c + [-side,-side] ), np.round( l_c + [ side,-side] ), np.round( l_c + [ side, side] ) ] ) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) return mat """