Spaces:
Sleeping
Sleeping
File size: 15,635 Bytes
95e98bb 3728629 95e98bb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 |
import numpy as np
import cv2
from os.path import join
import dlib
import os
from PIL import Image
import urllib.request
import imageio
width_ratio = 1.5
top_ratio = 1.5
gap_ratio = 0.1
down_ratio = 4.5
chin_width_ratio = 2.8
forehead_ratio = 0.3
verb = False
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PREDICTOR_PATH = os.path.join(BASE_DIR, "shape_predictor_68_face_landmarks.dat")
eye_cascade = cv2.CascadeClassifier(os.path.join(BASE_DIR, "haarcascade_eye.xml"))
assert not eye_cascade.empty()
SCALE_FACTOR = 1
FEATHER_AMOUNT = 11
FACE_POINTS = list(range(17, 68))
MOUTH_POINTS = list(range(48, 61))
RIGHT_BROW_POINTS = list(range(17, 22))
LEFT_BROW_POINTS = list(range(22, 27))
RIGHT_EYE_POINTS = list(range(36, 42))
LEFT_EYE_POINTS = list(range(42, 48))
NOSE_POINTS = list(range(27, 35))
JAW_POINTS = list(range(0, 17))
OVERLAY_POINTS = [
LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS,
NOSE_POINTS + MOUTH_POINTS,
]
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(PREDICTOR_PATH)
class TooManyFaces(Exception):
pass
class NoFaces(Exception):
pass
def get_landmarks(im):
rects = detector(im, 1)
if len(rects) > 1:
raise TooManyFaces
if len(rects) == 0:
raise NoFaces
return np.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])
def read_imgURL(URL):
with urllib.request.urlopen(URL) as url:
with open('temp.jpg', 'wb') as f:
f.write(url.read())
img = Image.open('temp.jpg')
img = np.array(img)
return img
def draw_convex_hull(im, points, color):
points = cv2.convexHull(points)
cv2.fillConvexPoly(im, points, color=color)
def get_face_mask(im, landmarks):
im = np.zeros(im.shape[:2], dtype=np.float64)
for group in OVERLAY_POINTS:
draw_convex_hull(im,
landmarks[group],
color=1)
im = np.array([im, im, im]).transpose((1, 2, 0))
im = (cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0) > 0) * 1.0
im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)
return im
def read_im_and_landmarks(fname):
im = np.array(fname)
im = cv2.resize(im, (im.shape[1] * SCALE_FACTOR,
im.shape[0] * SCALE_FACTOR))
s = get_landmarks(im)
return im, s
def warp_im(im, M, dshape):
output_im = np.zeros(dshape, dtype=im.dtype)
cv2.warpAffine(im,
M[:2],
(dshape[1], dshape[0]),
dst=output_im,
borderMode=cv2.BORDER_TRANSPARENT,
flags=cv2.WARP_INVERSE_MAP)
return output_im
def infer_chin_region(eye, width_ratio, down_ratio, left_or_right):
region1 = [0] * 4
if left_or_right == 'right': #assuming it is the absolute right chin
region1[0] = int(max(0, int(eye[0] - 0.5 * eye[2]))) #chin region should go lefwards
region1[2] = int(0.5 * eye[2])
else: # assuming it is the absolute left chin
region1[0] = int(eye[0] + eye[2]) # chin region should go rightwards
region1[2] = int(0.5 * eye[2])
region1[1] = int(eye[1] + eye[3])
region1[3] = int(1.5 * eye[3])
return region1
def detect_face_direction(gray, face, eye, down_ratio, chin_width_ratio):
region1 = [0] * 4 # assuming this is the left eye, forhead should go rightward
region2 = [0] * 4 # assuming this is the right eye, forhead should go leftward
print(eye[0])
region1 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'left') #region1 is from eye to right
region2 = infer_chin_region(eye[0], chin_width_ratio, down_ratio, 'right') # region2 is from eye to left
std1 = np.std(gray[region1[1]:(region1[1]+region1[3]), region1[0]:(region1[0]+region1[2])])
std2 = np.std(gray[region2[1]:(region2[1]+region2[3]), region2[0]:(region2[0]+region2[2])])
face_direction = ""
if std1 > std2: #eye right has higher variance than eye left
face_direction = "right"
else:
face_direction = "left"
return face_direction
def extract_cheek_region(face_x_min, face_x_max, face_y_max, eye_landmarks, left_or_right):
if left_or_right == "Left":
cheek_region_min_x = eye_landmarks[0,0]
cheek_region_max_x = int(face_x_max - 0.05 * (face_x_max - min(eye_landmarks[:,0])))
else:
cheek_region_max_x = max(eye_landmarks[:,0])[0,0]
#print (max(eye_landmarks[:,0])[0,0])
#cheek_region_max_x = max(eye_landmarks[:, 0])
cheek_region_min_x = int(face_x_min + 0.1 * (cheek_region_max_x - face_x_min))
cheek_region_min_y = int(max(eye_landmarks[:,1]) + 0.2 * (max(eye_landmarks[:,1]) - min(eye_landmarks[:,1])))
cheek_region_max_y = int(face_y_max - 0.1 * (face_y_max - max(eye_landmarks[:,1])))
return [cheek_region_min_x, cheek_region_min_y, cheek_region_max_x, cheek_region_max_y]
def extract_patches(imagefile, dimension_dict, face_loc_dict, image_dim, croppedFaces_Dir):
imageName = "temp"
img, landmarks = read_im_and_landmarks(imagefile)
face_detected = True
img_height, img_width = img.shape[0:2]
image_dim = [img_height, img_width]
min_dim = min(img_height, img_width)
min_face_size = min(min_dim * 0.2, min_dim * 0.2)
min_eye = min_face_size * 0.2
min_eye_area = min_eye ** 2
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
if face_detected:
mask = get_face_mask(img, landmarks)
face_x_min = int(max(0, np.asarray(min(landmarks[:,0])).flatten()[0]))
face_x_max = int(min(img_width, np.asarray(max(landmarks[:,0])).flatten()[0]))
face_y_min = int(max(0, np.asarray(min(landmarks[:,1])).flatten()[0]))
face_y_max = int(min(img_height, np.asarray(max(landmarks[:,1])).flatten()[0]))
face_loc_dict['face_loc'] = [face_x_min, face_x_max, face_y_min, face_y_max]
face_height = face_y_max - face_y_min
forehead_height = int(face_height * forehead_ratio)
new_face_y_min = max(0, face_y_min - forehead_height)
right_brow_landmarks = landmarks[RIGHT_BROW_POINTS,:]
left_brow_landmarks = landmarks[LEFT_BROW_POINTS,:]
right_eye_landmarks = landmarks[RIGHT_EYE_POINTS,:]
left_eye_landmarks = landmarks[LEFT_EYE_POINTS,:]
mouse_landmarks = landmarks[MOUTH_POINTS,:]
########################
# Get the forehead patch
########################
[right_brow_min_x, left_brow_max_x] = \
[max(0, np.min(np.array(right_brow_landmarks[:,0]))), min(img_width, np.max(np.array(left_brow_landmarks[:,0])))]
brow_min_y = min(np.min(np.array(right_brow_landmarks[:,1])),np.min(np.array(left_brow_landmarks[:,1])))
forehead_x_min = right_brow_min_x
forehead_x_max = left_brow_max_x
forehead_y_min = max(0, brow_min_y - forehead_height)
forehead_y_max = min(brow_min_y, forehead_y_min + forehead_height)
forehead_region = img[forehead_y_min:forehead_y_max, forehead_x_min:forehead_x_max, :]
#print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_x_min, forehead_x_max, forehead_y_min, forehead_y_max))
key_name = 'landmark_fh'
dimension_dict[key_name] = [forehead_x_min, forehead_x_max, forehead_y_min, forehead_y_max]
forehead_file_name = join(croppedFaces_Dir, key_name +".jpg")
#forehead_region = cv2.cvtColor(forehead_region, cv2.COLOR_BGR2RGB)
imageio.imwrite(forehead_file_name, forehead_region)
chin_x_min = np.max(np.array(right_eye_landmarks[:,0]))
chin_x_max = np.min(np.array(left_eye_landmarks[:,0]))
chin_y_min = np.max(np.array(mouse_landmarks[:,1]))
chin_y_max = face_y_max
chin_region = img[chin_y_min:chin_y_max, chin_x_min:chin_x_max, :]
#print ('chin dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (chin_x_min, chin_x_max, chin_y_min, chin_y_max))
key_name = 'landmark_chin'
dimension_dict[key_name] = [chin_x_min, chin_x_max, chin_y_min, chin_y_max]
chin_file_name = join(croppedFaces_Dir, key_name +".jpg")
#chin_region = cv2.cvtColor(chin_region, cv2.COLOR_BGR2RGB)
imageio.imwrite(chin_file_name, chin_region)
##########################
# Get the cheeks patch
##########################
# Decide whether it is a side view or not
left_eye_width = np.max(np.array(left_eye_landmarks[:,0])) - np.min(np.array(left_eye_landmarks[:,0]))
right_eye_width = np.max(np.array(right_eye_landmarks[:,0])) - np.min(np.array(right_eye_landmarks[:,0]))
right_face = True
left_face = True
if float(right_eye_width) / float(left_eye_width) >= 1.15: # right eye is bigger than left eye, showing the right face
left_face = False
elif float(left_eye_width) / float(right_eye_width) >= 1.15: # left eye is bigger than right eye, showing the left face
right_face = False
if right_face:
right_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, right_eye_landmarks, "Right")
cheek_region = img[right_cheek_region[1]:right_cheek_region[3], right_cheek_region[0]:right_cheek_region[2], :]
#print ('right cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3]))
key_name = 'landmark_rc'
dimension_dict[key_name] = [right_cheek_region[0], right_cheek_region[2], right_cheek_region[1], right_cheek_region[3]]
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
imageio.imwrite(cheek_file_name, cheek_region)
if left_face:
left_cheek_region = extract_cheek_region(face_x_min, face_x_max, face_y_max, left_eye_landmarks, "Left")
cheek_region = img[left_cheek_region[1]:left_cheek_region[3], left_cheek_region[0]:left_cheek_region[2], :]
#print ('left cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3]))
key_name = 'landmark_lc'
dimension_dict[key_name] = [left_cheek_region[0], left_cheek_region[2], left_cheek_region[1], left_cheek_region[3]]
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
imageio.imwrite(cheek_file_name, cheek_region)
if not face_detected:
print("Face not detected by landmarks model...")
# Use the OneEye model to detect one eye, and infer the face region based on the eye location
eye_detected = False
roi_gray = gray
roi_color = img
roi_color = cv2.cvtColor(roi_color, cv2.COLOR_BGR2RGB)
eyes = eye_cascade.detectMultiScale(roi_gray, 1.1, 5)
max_area = 0
eye_count = 0
max_index = 0
for (ex,ey,ew,eh) in eyes: # there might be multiple eyes detected. Choose the biggest one
if ew*eh >= max_area and ex >= img_width * 0.1 and ex <= img_width * 0.9:
max_area = ew*eh
max_index = eye_count
eye_count += 1
if max_area >= min_eye_area:
eye_detected = True
(ex, ey, ew, eh) = eyes[max_index]
if float(ew) / float(img_width) > 0.15 or float(eh) / float(img_height) > 0.15: # detected eye too large
# resize the detected eye
center_x = ex + ew/2
center_y = ey + eh/2
resized_w = min(img_width * 0.15, img_height * 0.15)
ex = int(center_x - resized_w/2)
ey = int(center_y - resized_w/2)
ew = int(resized_w)
eh = int(resized_w)
eyes1 = np.array([ex, ey, resized_w, resized_w]).reshape((1,4))
else:
eyes1 = np.array(eyes[max_index]).reshape((1,4))
face1 = np.array(())
face_direction = detect_face_direction(gray, face1, eyes1, down_ratio, chin_width_ratio)
if face_direction == "left":
print("Left eye detected")
face_min_x = eyes1[0, 0]
face_max_x = min(img_width, int(eyes1[0,0] + (chin_width_ratio + 0.5) * eyes1[0, 2]))
forehead_max_x = min(img_width, int(eyes1[0,0] + width_ratio * eyes1[0, 2]))
forehead_min_x = face_min_x
cheek_min_x = int(eyes1[0, 0] + 0.5 * eyes1[0,2])
cheek_max_x = face_max_x
else:
print("Right eye detected")
face_min_x = max(0, int(eyes1[0, 0] - chin_width_ratio * eyes1[0, 2]))
face_max_x = eyes1[0, 0] + eyes1[0, 2]
forehead_min_x = max(0, int(eyes1[0, 0] - width_ratio * eyes1[0, 2]))
forehead_max_x = min(img_width, int(eyes1[0, 0] + width_ratio * eyes1[0, 2]))
cheek_max_x = int(eyes1[0,0] + 0.5*eyes1[0,2])
cheek_min_x = face_min_x
forehead_min_y = max(0, int(eyes1[0, 1] - top_ratio * eyes1[0,3]))
forehead_max_y = max(0, int(eyes1[0, 1] - 0.5 * eyes1[0, 3]))
forehead_ok = False
# Get the forehead region
if forehead_max_y - forehead_min_y >= 0.7 * eyes1[0, 3]:
forehead_ok = True
forehead_region = img[forehead_min_y:forehead_max_y, forehead_min_x: forehead_max_x, :]
#print ('forehead dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (forehead_min_x, forehead_max_x, forehead_min_y, forehead_max_y))
key_name = 'oneeye_fh'
dimension_dict[key_name] = [forehead_min_x, forehead_max_x, forehead_min_y, forehead_max_y]
forehead_file_name = join(croppedFaces_Dir, key_name +".jpg")
imageio.imwrite(forehead_file_name, forehead_region)
# Get the cheek region
cheek_min_y = int(eyes1[0, 1] + eyes1[0, 3])
cheek_max_y = min(img_height, int(eyes1[0, 1] + down_ratio * eyes1[0, 3]))
cheek_region = img[cheek_min_y: cheek_max_y, cheek_min_x: cheek_max_x, :]
#print ('cheek dim (x_min, x_max, y_min, y_max): %i,%i, %i, %i' % (cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y))
key_name = 'oneeye_cheek'
dimension_dict[key_name] = [cheek_min_x, cheek_max_x, cheek_min_y, cheek_max_y]
face_loc_dict['face_loc'] = [face_min_x, face_max_x, forehead_min_y, cheek_max_y]
#cheek_region = cv2.cvtColor(cheek_region, cv2.COLOR_BGR2RGB)
if face_direction == "left":
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
elif face_direction == "right":
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
else:
cheek_file_name = join(croppedFaces_Dir, key_name +".jpg")
imageio.imwrite(cheek_file_name, cheek_region)
if (not face_detected) and (not eye_detected):
print("No chin or forehead detected, output the original file %s.jpg"%imageName)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
outfile = join(croppedFaces_Dir, imageName+".jpg")
imageio.imwrite(outfile, img)
return dimension_dict, face_loc_dict, image_dim |