Skip to content

Commit

Permalink
Add pose & expression estimation code
Browse files Browse the repository at this point in the history
  • Loading branch information
Tran Tuan Anh committed Jul 19, 2017
1 parent 8e75041 commit c4b0eb1
Show file tree
Hide file tree
Showing 1,697 changed files with 321,765 additions and 52 deletions.
Binary file added 3DMM_model/mod_struct_expr.mat
View file
Binary file not shown.
96 changes: 72 additions & 24 deletions trimBaselFace.py → 3DMM_model/trimBaselFace.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,31 @@
import numpy as np
import scipy.io
import os
import h5py

with_forehead = False

def convert(data):
new = data.reshape(data.shape[0],1).astype('float64')
return new
if not os.path.exists('01_MorphableModel.mat'):
print "Cannot find '01_MorphableModel.mat'"
exit(0)
model = scipy.io.loadmat('01_MorphableModel.mat',squeeze_me=True,struct_as_record=False)
mod_struct = scipy.io.loadmat('mod_struct.mat',squeeze_me=True,struct_as_record=False)

if not os.path.exists('Model_Expression.mat'):
print "Cannot find 'Model_Expression.mat'"
exit(0)

mod_struct = scipy.io.loadmat('mod_struct_expr.mat',squeeze_me=True,struct_as_record=False)
mod_struct = mod_struct['mod_struct'];

############## Basel Face Model ##########################################
model = scipy.io.loadmat('01_MorphableModel.mat',squeeze_me=True,struct_as_record=False)
##Python indexing
vertex_indices=mod_struct.vertex_indices
if not with_forehead:
vertex_indices=mod_struct.vertex_indices
else:
vertex_indices=mod_struct.vertex_indices_wForehead
numV_new = vertex_indices.shape[0]
##MATLAB : vertex_indices3 = [3*mod_struct.vertex_indices-2;3*mod_struct.vertex_indices-1;3*mod_struct.vertex_indices];
vertex_indices3 = [None] * (numV_new*3);
Expand All @@ -44,47 +57,82 @@ def convert(data):
shapeMU[0:-1:3] = shapeMU[0:-1:3] + mod_struct.tx
##MATLAB: BFM.shapeEV = model.shapeEV(1:99)/1000;
shapeEV = model['shapeEV'][0:99]/1000.
##MATLAB BFM.shapePC = model.shapePC(vertex_indices3,1:99);
##MATLAB: BFM.shapePC = model.shapePC(vertex_indices3,1:99);
shapePC = model['shapePC'][vertex_indices3,0:99]
##MATLAB BFM.texMU = model.texMU(vertex_indices3,:);
##MATLAB: BFM.texMU = model.texMU(vertex_indices3,:);
texMU = model['texMU'][vertex_indices3]
##MATLAB BFM.texPC = model.texPC(vertex_indices3,1:99);
##MATLAB: BFM.texPC = model.texPC(vertex_indices3,1:99);
texPC = model['texPC'][vertex_indices3,0:99];
##MATLAB BFM.texEV = model.texEV(1:99);
##MATLAB: BFM.texEV = model.texEV(1:99);
texEV = model['texEV'][0:99];
innerLandmarkIndex = mod_struct.innerLandmarkIndex
outerLandmarkIndex = mod_struct.outerLandmarkIndex
#% Trimming faces
#ind_inv = zeros(1,length(model.shapeMU)/3);
ind_inv = np.zeros(model['shapeMU'].shape[0]/3)
#ind_inv(mod_struct.vertex_indices) = 1: length(mod_struct.vertex_indices);
ind_inv[vertex_indices-1] = np.arange(1,vertex_indices.shape[0]+1)
#faces = ind_inv(model.tl);
faces = ind_inv[model['tl']-1]
#faces_keep = find(faces(:,1)>0 & faces(:,2)>0 & faces(:,3)>0);
faces_keep = ( faces[:,0]>0) & (faces[:,1]>0) & (faces[:,2]>0 )
#faces = faces(faces_keep,[2 1 3])
faces = np.dstack([faces[faces_keep,1], faces[faces_keep,0], faces[faces_keep,2]])
faces = faces[0]

if not with_forehead:
innerLandmarkIndex = mod_struct.innerLandmarkIndex
outerLandmarkIndex = mod_struct.outerLandmarkIndex
faces = mod_struct.faces
else:
innerLandmarkIndex = mod_struct.innerLandmarkIndex_wForehead
outerLandmarkIndex = mod_struct.outerLandmarkIndex_wForehead
faces = mod_struct.faces_wForehead

############## Expression Model #############################################
model_exp = scipy.io.loadmat('Model_Expression.mat',squeeze_me=True,struct_as_record=False)
##Python indexing
vertex_expr_ref=mod_struct.vertex_expr_ref
if not with_forehead:
vertex_indices_exp=[vertex_expr_ref[i-1] for i in mod_struct.vertex_indices]
else:
vertex_indices_exp=[vertex_expr_ref[i-1] for i in mod_struct.vertex_indices_wForehead]
##MATLAB : vertex_indices_exp3 = [3*mod_struct.vertex_indices_exp-2;3*mod_struct.vertex_indices_exp-1;3*mod_struct.vertex_indices_exp];
vertex_indices_exp3 = [None] * (numV_new*3);
vertex_indices_exp3[0::3] = [3*(i-1) for i in vertex_indices_exp];
vertex_indices_exp3[1::3] = [3*(i-1)+1 for i in vertex_indices_exp];
vertex_indices_exp3[2::3] = [3*(i-1)+2 for i in vertex_indices_exp];
vertex_indices_exp3 = np.array(vertex_indices_exp3)
##MATLAB: BFM.expMU = mu_exp(vertex_indices_exp3)/1000;
expMU = model_exp['mu_exp'][vertex_indices_exp3]/1000;
##MATLAB: BFM.expEV = mod_struct.expEV;
expEV = mod_struct.expEV;
##MATLAB: BFM.expPC = w_exp(vertex_indices_exp3,:)/1000;
expPC = model_exp['w_exp'][vertex_indices_exp3,:]/1000;

shapeEV = convert(shapeEV)
outerLandmarkIndex = convert(outerLandmarkIndex)
innerLandmarkIndex = convert(innerLandmarkIndex)
texEV = convert(texEV)
shapeMU = convert(shapeMU)
expMU = convert(expMU)
expEV = convert(expEV)


data = { 'BFM' : {
'faces' : faces,
'faces' : faces,
'shapeMU' : shapeMU,
'shapePC' : shapePC,
'shapeEV' : shapeEV,
'texMU' : texMU,
'texPC' : texPC,
'texEV' : texEV,
'expMU' : expMU,
'expPC' : expPC,
'expEV' : expEV,
'innerLandmarkIndex' : innerLandmarkIndex,
'outerLandmarkIndex' : outerLandmarkIndex,
}
}
scipy.io.savemat('BaselFaceModel_mod.mat',data)
if os.path.exists('01_MorphableModel.mat'):
os.remove('01_MorphableModel.mat')

fH5 = h5py.File('BaselFaceModel_mod.h5', "w")
fH5['faces'] = faces;
fH5['shapeMU'] = shapeMU;
fH5['shapePC'] = shapePC;
fH5['shapeEV'] = shapeEV;
fH5['texMU'] = texMU;
fH5['texPC'] = texPC;
fH5['texEV'] = texEV;
fH5['expMU'] = expMU;
fH5['expPC'] = expPC;
fH5['expEV'] = expEV;
fH5['innerLandmarkIndex'] = innerLandmarkIndex;
fH5['outerLandmarkIndex'] = outerLandmarkIndex;
fH5.close()
66 changes: 66 additions & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
cmake_minimum_required (VERSION 2.6)
project (CLM_framework)

list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:\$ORIGIN/../bin:\$ORIGIN")
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/)

# DEFINE PATH TO YOUR THIRD PARTY LIBs
set (DLIB_INCLUDE_DIR "/home/anh/Downloads/dlib-19.4")
set (DLIB_LIB_DIR "/home/anh/Downloads/dlib-19.4/shared_build/dlib")

# OpenCV
if(UNIX)
find_package( OpenCV REQUIRED )
MESSAGE("OpenCV information:")
MESSAGE(" OpenCV_INCLUDE_DIRS: ${OpenCV_INCLUDE_DIRS}")
MESSAGE(" OpenCV_LIBRARIES: ${OpenCV_LIBS}")
find_package( Boost REQUIRED COMPONENTS filesystem system)
MESSAGE("Boost information:")
MESSAGE(" Boost_INCLUDE_DIRS: ${Boost_INCLUDE_DIRS}")
MESSAGE(" Boost_LIBRARIES: ${Boost_LIBRARIES}")
MESSAGE(" Boost_LIBRARY_DIRS: ${Boost_LIBRARY_DIRS}")

INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}/boost)
LINK_DIRECTORIES(${Boost_LIBRARY_DIRS})
endif(UNIX)

if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
if (GCC_VERSION VERSION_LESS 4.7)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
else ()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
endif ()
else ()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
endif ()

# Boost
if(WIN32)
include_directories(lib/3rdParty/boost)
include_directories(lib/3rdParty/boost/boost)
link_directories( ${PROJECT_SOURCE_DIR}/lib/3rdParty/boost/lib )
else()
INCLUDE_DIRECTORIES(${BOOST_INCLUDE_DIR})
endif()

# OpenMP
find_package(OpenMP)
if (OPENMP_FOUND)
message("OPENMP FOUND")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
endif()

# Eigen
include_directories(lib/3rdParty/Eigen)

# libraries (ordering matters)
add_subdirectory(modules/CvGl)
add_subdirectory(modules/PoseExpr)

# executables
add_subdirectory(config)
0 dlib_model/.empty → CNN/empty
View file
File renamed without changes.
112 changes: 94 additions & 18 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
Python code for estimating 3DMM parameters using **[our very deep neural network](http://www.openu.ac.il/home/hassner/projects/CNN3DMM)**
Python code for 3D face modeling from single image using **[our very deep neural network](http://www.openu.ac.il/home/hassner/projects/CNN3DMM)**
===========
**_New_: Our code now supports face pose and expression fitting!**

This page contains end-to-end demo code that estimates the 3D facial shape and texture directly from an unconstrained 2D face image. For a given input image, it produces a standard ply file of the face shape and texture. It accompanies the deep network described in our paper [1].
This page contains end-to-end demo code that estimates the 3D facial shape and texture directly from an unconstrained 2D face image. For a given input image, it produces a standard ply file of the face shape and texture. It accompanies the deep network described in our paper [1]. We also include demo code of pose and expression fitting from landmarks in this release.

This release is part of an on-going face recognition and modeling project. Please, see **[this page](http://www.openu.ac.il/home/hassner/projects/CNN3DMM)** for updates and more data.

**A new version of this code which includes also pose and expression fitting, is being prepared and will be released soon.**

![Teaser](http://www-bcf.usc.edu/~iacopoma/img/3dmm_code_teaser.png)


Expand All @@ -15,38 +14,75 @@ This release is part of an on-going face recognition and modeling project. Pleas
* Designed and tested on **face images in unconstrained conditions**, including the challenging LFW, YTF and IJB-A benchmarks
* The 3D face shape and texture parameters extracted using our network were **shown for the first time to be descriminative and robust**, providing near state of the art face recognition performance with 3DMM representations on these benchmarks
* **No expensive, iterative optimization, inner loops** to regress the shape. 3DMM fitting is therefore extremely fast
* Extra code for **head pose and expression estimation from detected facial landmarks**, with the use of the regressed 3D face model

## Dependencies

## Library requirements

* [Dlib Python Wrapper](http://dlib.net/)
* [OpenCV Python Wrapper](http://opencv.org/)
* [Dlib Python and C++ library](http://dlib.net/)
* [OpenCV Python and C++ library](http://opencv.org/)
* [Caffe](caffe.berkeleyvision.org) (**version 1.0.0-rc3 or above required**)
* [Numpy](http://www.numpy.org/)
* [Python2.7](https://www.python.org/download/releases/2.7/)

The code has been tested on Linux only. On Linux you can rely on the default version of python, installing all the packages needed from the package manager or on Anaconda Python and install required packages through `conda`. A bit more effort is required to install caffé.
The code has been tested on Linux only. On Linux you can rely on the default version of python, installing all the packages needed from the package manager or on Anaconda Python and install required packages through `conda`. A bit more effort is required to install caffé, dlib, and libhdf5.

## Data requirements

Before running the code, please, make sure to have all the required data in the following specific folder:
- **[Download our CNN](http://www.openu.ac.il/home/hassner/projects/CNN3DMM)**
- **[Download the Basel Face Model](http://faces.cs.unibas.ch/bfm/main.php?nav=1-2&id=downloads)**
- **[Download our CNN](http://www.openu.ac.il/home/hassner/projects/CNN3DMM)** and move the CNN model (3 files: `3dmm_cnn_resnet_101.caffemodel`,`deploy_network.prototxt`,`mean.binaryproto`) into the `CNN` folder
- **[Download the Basel Face Model](http://faces.cs.unibas.ch/bfm/main.php?nav=1-2&id=downloads)** and move `01_MorphableModel.mat` into the `3DMM_model` folder
- **[Acquire 3DDFA Expression Model](http://www.cbsr.ia.ac.cn/users/xiangyuzhu/projects/3DDFA/Code/3DDFA.zip)**, run its code to generate `Model_Expression.mat` and move this file the `3DMM_model` folder
- Go into `3DMM_model` folder. Run the script `python trimBaselFace.py`. This should output 2 files `BaselFaceModel_mod.mat` and `BaselFaceModel_mod.h5`.
- **[Download dlib face prediction model](http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2)** and move the `.dat` file into the `dlib_model` folder.
- Move the CNN model (3 files: `3dmm_cnn_resnet_101.caffemodel`,`deploy_network.prototxt`,`mean.binaryproto`) into the `CNN` folder
- Copy the Basel Face Model (`01_MorphableModel.mat`) in the same folder of `demo.py` file.
- Run the script `python trimBaselFace.py`. This should output a file `BaselFaceModel_mod.mat` and remove the original one automatically.

## Installation (pose & expression fitting code)

- Install **cmake**:
```
apt-get install cmake
```
- Install **opencv** (2.4.6 or higher is recommended):
```
(http://docs.opencv.org/doc/tutorials/introduction/linux_install/linux_install.html)
```
- Install **libboost** (1.5 or higher is recommended):
```
apt-get install libboost-all-dev
```
- Install **OpenGL, freeglut, and glew**
```
sudo apt-get install freeglut3-dev
sudo apt-get install libglew-dev
```
- Install **libhdf5-dev** library
```
sudo apt-get install libhdf5-dev
```
- Install **Dlib C++ library**
```
(http://dlib.net/)
```
- Update Dlib directory paths (`DLIB_INCLUDE_DIR` and `DLIB_LIB_DIR`) in `CMakeLists.txt`
- Make build directory (temporary). Make & install to bin folder
```
mkdir build
cd build
cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=../bin ..
make
make install
```
This code should generate `TestVisualization` in `bin` folder

## Usage

### Input
### 3DMM fitting on a set of input images

The demo script can be used from the command line with the following syntax:
* Go into `demoCode` folder. The demo script can be used from the command line with the following syntax:

```bash
$ Usage: python demo.py <inputList> <outputDir> <needCrop> <useLM>
$ Usage: python testBatchModel.py <inputList> <outputDir> <needCrop> <useLM>
```

where the parameters are the following:
Expand All @@ -62,11 +98,10 @@ data/2.jpg
....
</pre>

### Output
The demo code should produce an output similar to this:
* The demo code should produce an output similar to this:

```bash
user@system:~/Desktop/3dmm_release$ python demo.py input.txt out/
user@system:~/Desktop/3dmm_release$ python testBatchModel.py input.txt out/
> Prepare image data/1.jpg:
> Number of faces detected: 1
> Prepare image data/2.jpg:
Expand All @@ -90,6 +125,47 @@ which should produce something similar to:

![Teaser](http://www-bcf.usc.edu/~iacopoma/img/meshlab_disp.png)


### 3D Face modeling + pose & expression estimation on a single input image

* Go into `demoCode` folder. The demo script can be used from the command line with the following syntax:

```bash
$ Usage: python testModel_PoseExpr.py <outputDir> <save3D>
```

where the parameters are the following:
- `<outputDir>` is the path to the output directory, where 3DMM (and ply) files are stored.
- `<save3D>` is an option to save the ply file (1) or not (0). Default 1.

* The program will pop up a dialog to select an input image. Then it will estimate 3DMM paramters (with the CNN model), estimate pose+expression and visualize the result (with C++ program)

Example:
```bash
user@system:~/Desktop/3dmm_release$ python testModel_PoseExpr.py out/
(Select `Anders_Fogh_Rasmussen_0004.jpg`)
> Prepare image /home/anh/Downloads/PoseExprFromLM-master/demoCode/data/Anders_Fogh_Rasmussen_0004.jpg:
Number of faces detected: 1
> CNN Model loaded to regress 3D Shape and Texture!
> Loaded the Basel Face Model to write the 3D output!
*****************************************
** Caffe loading : 1.007 s
** Image cropping : 0.069 s
** 3D Modeling : 1.145 s
*****************************************
> Writing 3D file in: out/Anders_Fogh_Rasmussen_0004.ply
> Pose & expression estimation
load ../3DMM_model/BaselFaceModel_mod.h5
** Pose+expr fitting: 0.153 s
** Visualization : 0.052 s
*****************************************
```

The pop up window should look similar to:
![Teaser](https://sites.google.com/site/anhttranusc/PoseExpr_Demo.png)



## Citation

If you find this work useful, please cite our paper [1] with the following bibtex:
Expand Down
0 out/.empty → bin/empty
View file
File renamed without changes.
Loading

0 comments on commit c4b0eb1

Please sign in to comment.