How to Swap Faces in Images with Python: A Step‑by‑Step Guide

Introduction

In this article I will introduce how to write a short (200‑line) Python script that automatically swaps the face in one image with the face in another image.

1. Using dlib to Extract Facial Landmarks

The script uses dlib's Python bindings to extract facial landmarks:

import cv2
import dlib
import numpy as np
import sys

PREDICTOR_PATH = "/home/matt/dlib-18.16/shape_predictor_68_face_landmarks.dat"
SCALE_FACTOR = 1
FEATHER_AMOUNT = 11

# Points used to line up the images.
ALIGN_POINTS = list(range(17, 68))

# Points from the second image to overlay on the first. The convex hull of each element will be overlaid.
OVERLAY_POINTS = [
    list(range(42, 48)) + list(range(36, 42)) + list(range(22, 27)) + list(range(17, 22)),
    list(range(27, 36)) + list(range(48, 61))
]

COLOUR_CORRECT_BLUR_FRAC = 0.6

detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(PREDICTOR_PATH)

class TooManyFaces(Exception):
    pass

class NoFaces(Exception):
    pass

def get_landmarks(im):
    rects = detector(im, 1)
    if len(rects) > 1:
        raise TooManyFaces
    if len(rects) == 0:
        raise NoFaces
    return np.matrix([[p.x, p.y] for p in predictor(im, rects[0]).parts()])

def transformation_from_points(points1, points2):
    """Return an affine transformation [s * R | T] such that:
    sum ||s*R*p1,i + T - p2,i||^2 is minimized.
    """
    points1 = points1.astype(np.float64)
    points2 = points2.astype(np.float64)
    c1 = np.mean(points1, axis=0)
    c2 = np.mean(points2, axis=0)
    points1 -= c1
    points2 -= c2
    s1 = np.std(points1)
    s2 = np.std(points2)
    points1 /= s1
    points2 /= s2
    U, S, Vt = np.linalg.svd(points1.T * points2)
    R = (U * Vt).T
    return np.vstack([np.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)), np.matrix([0., 0., 1.])])

def warp_im(im, M, dshape):
    output_im = np.zeros(dshape, dtype=im.dtype)
    cv2.warpAffine(im, M[:2], (dshape[1], dshape[0]), dst=output_im,
                   borderMode=cv2.BORDER_TRANSPARENT, flags=cv2.WARP_INVERSE_MAP)
    return output_im

def correct_colours(im1, im2, landmarks1):
    blur_amount = COLOUR_CORRECT_BLUR_FRAC * np.linalg.norm(
        np.mean(landmarks1[LEFT_EYE_POINTS], axis=0) - np.mean(landmarks1[RIGHT_EYE_POINTS], axis=0))
    blur_amount = int(blur_amount)
    if blur_amount % 2 == 0:
        blur_amount += 1
    im1_blur = cv2.GaussianBlur(im1, (blur_amount, blur_amount), 0)
    im2_blur = cv2.GaussianBlur(im2, (blur_amount, blur_amount), 0)
    im2_blur = im2_blur + 128 * (im2_blur <= 1.0)
    return (im2.astype(np.float64) * (im1_blur / im2_blur)).astype(np.uint8)

def get_face_mask(im, landmarks):
    im = np.zeros(im.shape[:2], dtype=np.float64)
    for group in OVERLAY_POINTS:
        draw_convex_hull(im, landmarks[group], 1)
    im = cv2.GaussianBlur(im, (FEATHER_AMOUNT, FEATHER_AMOUNT), 0)
    im = np.array(im > 0, dtype=np.uint8)
    return im

def draw_convex_hull(im, points, color):
    points = cv2.convexHull(points)
    cv2.fillConvexPoly(im, points, color)

def read_im_and_landmarks(fname):
    im = cv2.imread(fname, cv2.IMREAD_COLOR)
    im = cv2.resize(im, (int(im.shape[1] * SCALE_FACTOR), int(im.shape[0] * SCALE_FACTOR)))
    s = get_landmarks(im)
    return im, s

if __name__ == "__main__":
    im1, landmarks1 = read_im_and_landmarks(sys.argv[1])
    im2, landmarks2 = read_im_and_landmarks(sys.argv[2])
    M = transformation_from_points(landmarks1[ALIGN_POINTS], landmarks2[ALIGN_POINTS])
    mask = get_face_mask(im2, landmarks2)
    warped_mask = warp_im(mask, M, im1.shape)
    combined_mask = np.max([get_face_mask(im1, landmarks1), warped_mask], axis=0)
    warped_im2 = warp_im(im2, M, im1.shape)
    warped_corrected_im2 = correct_colours(im1, warped_im2, landmarks1)
    output_im = im1 * (1.0 - combined_mask) + warped_corrected_im2 * combined_mask
    cv2.imwrite('output.jpg', output_im)

2. Align Faces Using Procrustes Analysis

Now that we have two landmark matrices (each row corresponds to a specific facial feature), we need to find a rotation, translation, and scaling that best aligns the first set of points to the second. This is solved using ordinary Procrustes analysis, which computes the optimal affine transformation.

3. Color Correct the Second Image

Directly overlaying facial features often reveals mismatched skin tones and lighting. To mitigate this, we adjust the colors of the second image by scaling each pixel based on the ratio of Gaussian‑blurred versions of the two images, using a blur radius proportional to the inter‑pupillary distance.

4. Blend Features of the Second Image into the First

We create a mask that defines where the second image should appear. Pixels with a mask value of 1 show the second image, 0 shows the first, and intermediate values blend the two. The mask is built from convex hulls around the eyes, eyebrows, nose, and mouth, then feathered to smooth edges.

Full Code (link)

See the complete script above.

Compiled by

Python Developer – LynnShaw (English translation by Matthew Earl)

Source: http://python.jobbole.com/82546/