123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426 |
- """This script defines the parametric 3d face model for Deep3DFaceRecon_pytorch
- """
- import numpy as np
- import torch
- import torch.nn.functional as F
- from scipy.io import loadmat
- import os
- # from utils.commons.tensor_utils import convert_like
- def perspective_projection(focal, center):
- # return p.T (N, 3) @ (3, 3)
- return np.array([
- focal, 0, center,
- 0, focal, center,
- 0, 0, 1
- ]).reshape([3, 3]).astype(np.float32).transpose() # 注意这里的transpose!
- class SH:
- def __init__(self):
- self.a = [np.pi, 2 * np.pi / np.sqrt(3.), 2 * np.pi / np.sqrt(8.)]
- self.c = [1/np.sqrt(4 * np.pi), np.sqrt(3.) / np.sqrt(4 * np.pi), 3 * np.sqrt(5.) / np.sqrt(12 * np.pi)]
- class ParametricFaceModel:
- def __init__(self,
- bfm_folder='./BFM',
- recenter=True,
- camera_distance=10.,
- init_lit=np.array([
- 0.8, 0, 0, 0, 0, 0, 0, 0, 0
- ]),
- focal=1015.,
- center=112.,
- is_train=True,
- default_name='BFM_model_front.mat',
- keypoint_mode='mediapipe'):
-
- model = loadmat(os.path.join(bfm_folder, default_name))
- # mean face shape. [3*N,1]
- self.mean_shape = model['meanshape'].astype(np.float32)
- # identity basis. [3*N,80]
- self.id_base = model['idBase'].astype(np.float32)
- # expression basis. [3*N,64]
- self.exp_base = model['exBase'].astype(np.float32)
- # mean face texture. [3*N,1] (0-255)
- self.mean_tex = model['meantex'].astype(np.float32)
- # texture basis. [3*N,80]
- self.tex_base = model['texBase'].astype(np.float32)
- # face indices for each vertex that lies in. starts from 0. [N,8]
- self.point_buf = model['point_buf'].astype(np.int64) - 1
- # vertex indices for each face. starts from 0. [F,3]
- self.face_buf = model['tri'].astype(np.int64) - 1
- # vertex indices for 68 landmarks. starts from 0. [68,1]
- if keypoint_mode == 'mediapipe':
- self.keypoints = np.load("deep_3drecon/BFM/index_mp468_from_mesh35709.npy").astype(np.int64)
- unmatch_mask = self.keypoints < 0
- self.keypoints[unmatch_mask] = 0
- else:
- self.keypoints = np.squeeze(model['keypoints']).astype(np.int64) - 1
- if is_train:
- # vertex indices for small face region to compute photometric error. starts from 0.
- self.front_mask = np.squeeze(model['frontmask2_idx']).astype(np.int64) - 1
- # vertex indices for each face from small face region. starts from 0. [f,3]
- self.front_face_buf = model['tri_mask2'].astype(np.int64) - 1
- # vertex indices for pre-defined skin region to compute reflectance loss
- self.skin_mask = np.squeeze(model['skinmask'])
-
- if recenter:
- mean_shape = self.mean_shape.reshape([-1, 3])
- mean_shape = mean_shape - np.mean(mean_shape, axis=0, keepdims=True)
- self.mean_shape = mean_shape.reshape([-1, 1])
- self.key_mean_shape = self.mean_shape.reshape([-1, 3])[self.keypoints, :].reshape([-1, 3])
- self.key_id_base = self.id_base.reshape([-1, 3,80])[self.keypoints, :].reshape([-1, 80])
- self.key_exp_base = self.exp_base.reshape([-1, 3, 64])[self.keypoints, :].reshape([-1, 64])
- self.focal = focal
- self.center = center
- self.persc_proj = perspective_projection(focal, center)
- self.device = 'cpu'
- self.camera_distance = camera_distance
- self.SH = SH()
- self.init_lit = init_lit.reshape([1, 1, -1]).astype(np.float32)
- self.initialized = False
- def to(self, device):
- self.device = device
- for key, value in self.__dict__.items():
- if type(value).__module__ == np.__name__:
- setattr(self, key, torch.tensor(value).to(device))
- self.initialized = True
- return self
-
- def compute_shape(self, id_coeff, exp_coeff):
- """
- Return:
- face_shape -- torch.tensor, size (B, N, 3)
- Parameters:
- id_coeff -- torch.tensor, size (B, 80), identity coeffs
- exp_coeff -- torch.tensor, size (B, 64), expression coeffs
- """
- batch_size = id_coeff.shape[0]
- id_part = torch.einsum('ij,aj->ai', self.id_base, id_coeff)
- exp_part = torch.einsum('ij,aj->ai', self.exp_base, exp_coeff)
- face_shape = id_part + exp_part + self.mean_shape.reshape([1, -1])
- return face_shape.reshape([batch_size, -1, 3])
-
- def compute_key_shape(self, id_coeff, exp_coeff):
- """
- Return:
- face_shape -- torch.tensor, size (B, N, 3)
- Parameters:
- id_coeff -- torch.tensor, size (B, 80), identity coeffs
- exp_coeff -- torch.tensor, size (B, 64), expression coeffs
- """
- batch_size = id_coeff.shape[0]
- id_part = torch.einsum('ij,aj->ai', self.key_id_base, id_coeff)
- exp_part = torch.einsum('ij,aj->ai', self.key_exp_base, exp_coeff)
- face_shape = id_part + exp_part + self.key_mean_shape.reshape([1, -1])
- return face_shape.reshape([batch_size, -1, 3])
- def compute_texture(self, tex_coeff, normalize=True):
- """
- Return:
- face_texture -- torch.tensor, size (B, N, 3), in RGB order, range (0, 1.)
- Parameters:
- tex_coeff -- torch.tensor, size (B, 80)
- """
- batch_size = tex_coeff.shape[0]
- face_texture = torch.einsum('ij,aj->ai', self.tex_base, tex_coeff) + self.mean_tex
- if normalize:
- face_texture = face_texture / 255.
- return face_texture.reshape([batch_size, -1, 3])
- def compute_norm(self, face_shape):
- """
- Return:
- vertex_norm -- torch.tensor, size (B, N, 3)
- Parameters:
- face_shape -- torch.tensor, size (B, N, 3)
- """
- v1 = face_shape[:, self.face_buf[:, 0]]
- v2 = face_shape[:, self.face_buf[:, 1]]
- v3 = face_shape[:, self.face_buf[:, 2]]
- e1 = v1 - v2
- e2 = v2 - v3
- face_norm = torch.cross(e1, e2, dim=-1)
- face_norm = F.normalize(face_norm, dim=-1, p=2)
- face_norm = torch.cat([face_norm, torch.zeros(face_norm.shape[0], 1, 3).to(self.device)], dim=1)
-
- vertex_norm = torch.sum(face_norm[:, self.point_buf], dim=2)
- vertex_norm = F.normalize(vertex_norm, dim=-1, p=2)
- return vertex_norm
- def compute_color(self, face_texture, face_norm, gamma):
- """
- Return:
- face_color -- torch.tensor, size (B, N, 3), range (0, 1.)
- Parameters:
- face_texture -- torch.tensor, size (B, N, 3), from texture model, range (0, 1.)
- face_norm -- torch.tensor, size (B, N, 3), rotated face normal
- gamma -- torch.tensor, size (B, 27), SH coeffs
- """
- batch_size = gamma.shape[0]
- v_num = face_texture.shape[1]
- a, c = self.SH.a, self.SH.c
- gamma = gamma.reshape([batch_size, 3, 9])
- gamma = gamma + self.init_lit
- gamma = gamma.permute(0, 2, 1)
- Y = torch.cat([
- a[0] * c[0] * torch.ones_like(face_norm[..., :1]).to(self.device),
- -a[1] * c[1] * face_norm[..., 1:2],
- a[1] * c[1] * face_norm[..., 2:],
- -a[1] * c[1] * face_norm[..., :1],
- a[2] * c[2] * face_norm[..., :1] * face_norm[..., 1:2],
- -a[2] * c[2] * face_norm[..., 1:2] * face_norm[..., 2:],
- 0.5 * a[2] * c[2] / np.sqrt(3.) * (3 * face_norm[..., 2:] ** 2 - 1),
- -a[2] * c[2] * face_norm[..., :1] * face_norm[..., 2:],
- 0.5 * a[2] * c[2] * (face_norm[..., :1] ** 2 - face_norm[..., 1:2] ** 2)
- ], dim=-1)
- r = Y @ gamma[..., :1]
- g = Y @ gamma[..., 1:2]
- b = Y @ gamma[..., 2:]
- face_color = torch.cat([r, g, b], dim=-1) * face_texture
- return face_color
- @staticmethod
- def compute_rotation(angles, device='cpu'):
- """
- Return:
- rot -- torch.tensor, size (B, 3, 3) pts @ trans_mat
- Parameters:
- angles -- torch.tensor, size (B, 3), radian
- """
- batch_size = angles.shape[0]
- angles = angles.to(device)
- ones = torch.ones([batch_size, 1]).to(device)
- zeros = torch.zeros([batch_size, 1]).to(device)
- x, y, z = angles[:, :1], angles[:, 1:2], angles[:, 2:],
-
- rot_x = torch.cat([
- ones, zeros, zeros,
- zeros, torch.cos(x), -torch.sin(x),
- zeros, torch.sin(x), torch.cos(x)
- ], dim=1).reshape([batch_size, 3, 3])
-
- rot_y = torch.cat([
- torch.cos(y), zeros, torch.sin(y),
- zeros, ones, zeros,
- -torch.sin(y), zeros, torch.cos(y)
- ], dim=1).reshape([batch_size, 3, 3])
- rot_z = torch.cat([
- torch.cos(z), -torch.sin(z), zeros,
- torch.sin(z), torch.cos(z), zeros,
- zeros, zeros, ones
- ], dim=1).reshape([batch_size, 3, 3])
- rot = rot_z @ rot_y @ rot_x
- return rot.permute(0, 2, 1)
- def to_camera(self, face_shape):
- face_shape[..., -1] = self.camera_distance - face_shape[..., -1] # reverse the depth axis, add a fixed offset of length
- return face_shape
- def to_image(self, face_shape):
- """
- Return:
- face_proj -- torch.tensor, size (B, N, 2), y direction is opposite to v direction
- Parameters:
- face_shape -- torch.tensor, size (B, N, 3)
- """
- # to image_plane
- face_proj = face_shape @ self.persc_proj
- face_proj = face_proj[..., :2] / face_proj[..., 2:]
- return face_proj
- def transform(self, face_shape, rot, trans):
- """
- Return:
- face_shape -- torch.tensor, size (B, N, 3) pts @ rot + trans
- Parameters:
- face_shape -- torch.tensor, si≥ze (B, N, 3)
- rot -- torch.tensor, size (B, 3, 3)
- trans -- torch.tensor, size (B, 3)
- """
- return face_shape @ rot + trans.unsqueeze(1)
- def get_landmarks(self, face_proj):
- """
- Return:
- face_lms -- torch.tensor, size (B, 68, 2)
- Parameters:
- face_proj -- torch.tensor, size (B, N, 2)
- """
- return face_proj[:, self.keypoints]
- def split_coeff(self, coeffs):
- """
- Return:
- coeffs_dict -- a dict of torch.tensors
- Parameters:
- coeffs -- torch.tensor, size (B, 256)
- """
- id_coeffs = coeffs[:, :80]
- exp_coeffs = coeffs[:, 80: 144]
- tex_coeffs = coeffs[:, 144: 224]
- angles = coeffs[:, 224: 227]
- gammas = coeffs[:, 227: 254]
- translations = coeffs[:, 254:]
- return {
- 'id': id_coeffs,
- 'exp': exp_coeffs,
- 'tex': tex_coeffs,
- 'angle': angles,
- 'gamma': gammas,
- 'trans': translations
- }
- def compute_for_render(self, coeffs):
- """
- Return:
- face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- face_color -- torch.tensor, size (B, N, 3), in RGB order
- landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- Parameters:
- coeffs -- torch.tensor, size (B, 257)
- """
- coef_dict = self.split_coeff(coeffs)
- face_shape = self.compute_shape(coef_dict['id'], coef_dict['exp'])
- rotation = self.compute_rotation(coef_dict['angle'], device=self.device)
- face_shape_transformed = self.transform(face_shape, rotation, coef_dict['trans'])
- face_vertex = self.to_camera(face_shape_transformed)
-
- face_proj = self.to_image(face_vertex)
- landmark = self.get_landmarks(face_proj)
- face_texture = self.compute_texture(coef_dict['tex'])
- face_norm = self.compute_norm(face_shape)
- face_norm_roted = face_norm @ rotation
- face_color = self.compute_color(face_texture, face_norm_roted, coef_dict['gamma'])
- return face_vertex, face_texture, face_color, landmark
- def compute_face_vertex(self, id, exp, angle, trans):
- """
- Return:
- face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- face_color -- torch.tensor, size (B, N, 3), in RGB order
- landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- Parameters:
- coeffs -- torch.tensor, size (B, 257)
- """
- if not self.initialized:
- self.to(id.device)
- face_shape = self.compute_shape(id, exp)
- rotation = self.compute_rotation(angle, device=self.device)
- face_shape_transformed = self.transform(face_shape, rotation, trans)
- face_vertex = self.to_camera(face_shape_transformed)
- return face_vertex
-
- def compute_for_landmark_fit(self, id, exp, angles, trans, ret=None):
- """
- Return:
- face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- face_color -- torch.tensor, size (B, N, 3), in RGB order
- landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- Parameters:
- coeffs -- torch.tensor, size (B, 257)
- """
- face_shape = self.compute_key_shape(id, exp)
- rotation = self.compute_rotation(angles, device=self.device)
- face_shape_transformed = self.transform(face_shape, rotation, trans)
- face_vertex = self.to_camera(face_shape_transformed)
-
- face_proj = self.to_image(face_vertex)
- landmark = face_proj
- return landmark
- def compute_for_landmark_fit_nerf(self, id, exp, angles, trans, ret=None):
- """
- Return:
- face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- face_color -- torch.tensor, size (B, N, 3), in RGB order
- landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- Parameters:
- coeffs -- torch.tensor, size (B, 257)
- """
- face_shape = self.compute_key_shape(id, exp)
- rotation = self.compute_rotation(angles, device=self.device)
- face_shape_transformed = self.transform(face_shape, rotation, trans)
- face_vertex = face_shape_transformed # no to_camera
-
- face_proj = self.to_image(face_vertex)
- landmark = face_proj
- return landmark
- # def compute_for_landmark_fit(self, id, exp, angles, trans, ret={}):
- # """
- # Return:
- # face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- # face_color -- torch.tensor, size (B, N, 3), in RGB order
- # landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- # Parameters:
- # coeffs -- torch.tensor, size (B, 257)
- # """
- # face_shape = self.compute_shape(id, exp)
- # rotation = self.compute_rotation(angles)
- # face_shape_transformed = self.transform(face_shape, rotation, trans)
- # face_vertex = self.to_camera(face_shape_transformed)
-
- # face_proj = self.to_image(face_vertex)
- # landmark = self.get_landmarks(face_proj)
- # return landmark
-
- def compute_for_render_fit(self, id, exp, angles, trans, tex, gamma):
- """
- Return:
- face_vertex -- torch.tensor, size (B, N, 3), in camera coordinate
- face_color -- torch.tensor, size (B, N, 3), in RGB order
- landmark -- torch.tensor, size (B, 68, 2), y direction is opposite to v direction
- Parameters:
- coeffs -- torch.tensor, size (B, 257)
- """
- face_shape = self.compute_shape(id, exp)
- rotation = self.compute_rotation(angles, device=self.device)
- face_shape_transformed = self.transform(face_shape, rotation, trans)
- face_vertex = self.to_camera(face_shape_transformed)
-
- face_proj = self.to_image(face_vertex)
- landmark = self.get_landmarks(face_proj)
- face_texture = self.compute_texture(tex)
- face_norm = self.compute_norm(face_shape)
- face_norm_roted = face_norm @ rotation
- face_color = self.compute_color(face_texture, face_norm_roted, gamma)
- return face_color, face_vertex, landmark
|