Separate IO dependencies

OpenCV, LMDB, LevelDB and Snappy are made optional via switches
(USE_OPENCV, USE_LMDB, USE_LEVELDB) available for Make and CMake
builds. Since Snappy is a LevelDB dependency, its use is determined by
USE_LEVELDB. HDF5 is left bundled because it is used for serializing
weights and solverstates.

Author: eelstork

.travis.yml

@@ -2,11 +2,12 @@
 # one using CMake, and one using make.
 env:
   matrix:
-    - WITH_CUDA=false WITH_CMAKE=false
-    - WITH_CUDA=false WITH_CMAKE=true
-    - WITH_CUDA=true WITH_CMAKE=false
-    - WITH_CUDA=true WITH_CMAKE=true
-    - WITH_CUDA=false WITH_CMAKE=true PYTHON_VERSION=3
+    - WITH_CUDA=false WITH_CMAKE=false WITH_IO=true
+    - WITH_CUDA=false WITH_CMAKE=true WITH_IO=true PYTHON_VERSION=3
+    - WITH_CUDA=true WITH_CMAKE=false WITH_IO=true
+    - WITH_CUDA=true WITH_CMAKE=true WITH_IO=true
+    - WITH_CUDA=false WITH_CMAKE=false WITH_IO=false
+    - WITH_CUDA=false WITH_CMAKE=true WITH_IO=false PYTHON_VERSION=3
 
 language: cpp
 

CMakeLists.txt

@@ -16,13 +16,16 @@ include(cmake/ConfigGen.cmake)
 
 # ---[ Options
 caffe_option(CPU_ONLY  "Build Caffe without CUDA support" OFF) # TODO: rename to USE_CUDA
-caffe_option(USE_CUDNN "Build Caffe with cuDNN libary support" ON IF NOT CPU_ONLY)
+caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ONLY)
 caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON)
 caffe_option(BUILD_python "Build Python wrapper" ON)
 set(python_version "2" CACHE STRING "Specify which python version to use")
 caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
 caffe_option(BUILD_docs   "Build documentation" ON IF UNIX OR APPLE)
-caffe_option(BUILD_python_layer "Build the Caffe python layer" ON)
+caffe_option(BUILD_python_layer "Build the caffe python layer" ON)
+caffe_option(USE_LMDB "Build with lmdb" ON)
+caffe_option(USE_LEVELDB "Build with levelDB" ON)
+caffe_option(USE_OPENCV "Build with OpenCV support" ON)
 
 # ---[ Dependencies
 include(cmake/Dependencies.cmake)

Makefile

@@ -169,9 +169,18 @@ ifneq ($(CPU_ONLY), 1)
 	LIBRARY_DIRS += $(CUDA_LIB_DIR)
 	LIBRARIES := cudart cublas curand
 endif
-LIBRARIES += glog gflags protobuf leveldb snappy \
-	lmdb boost_system hdf5_hl hdf5 m \
-	opencv_core opencv_highgui opencv_imgproc
+
+LIBRARIES += glog gflags protobuf boost_system m hdf5_hl hdf5
+
+ifeq ($(USE_LEVELDB), 1)
+	LIBRARIES += leveldb snappy
+endif
+ifeq ($(USE_LMDB), 1)
+	LIBRARIES += lmdb
+endif
+ifeq ($(USE_OPENCV), 1)
+	LIBRARIES += opencv_core opencv_highgui opencv_imgproc
+endif
 PYTHON_LIBRARIES := boost_python python2.7
 WARNINGS := -Wall -Wno-sign-compare
 
@@ -290,6 +299,17 @@ ifeq ($(USE_CUDNN), 1)
 	COMMON_FLAGS += -DUSE_CUDNN
 endif
 
+# i/o libraries configuration
+ifeq ($(USE_OPENCV), 1)
+	COMMON_FLAGS += -DUSE_OPENCV
+endif
+ifeq ($(USE_LEVELDB), 1)
+	COMMON_FLAGS += -DUSE_LEVELDB
+endif
+ifeq ($(USE_LMDB), 1)
+	COMMON_FLAGS += -DUSE_LMDB
+endif
+
 # CPU-only configuration
 ifeq ($(CPU_ONLY), 1)
 	OBJS := $(PROTO_OBJS) $(CXX_OBJS)
@@ -472,7 +492,7 @@ runtest: $(TEST_ALL_BIN)
 
 pytest: py
 	cd python; python -m unittest discover -s caffe/test
-	
+
 mattest: mat
 	cd matlab; $(MATLAB_DIR)/bin/matlab -nodisplay -r 'caffe.run_tests(), exit()'
 

Makefile.config.example

@@ -7,6 +7,11 @@
 # CPU-only switch (uncomment to build without GPU support).
 # CPU_ONLY := 1
 
+# comment out to disable IO dependencies
+USE_LEVELDB := 1
+USE_LMDB := 1
+USE_OPENCV := 1
+
 # To customize your choice of compiler, uncomment and set the following.
 # N.B. the default for Linux is g++ and the default for OSX is clang++
 # CUSTOM_CXX := g++

cmake/ConfigGen.cmake

@@ -56,6 +56,18 @@ function(caffe_generate_export_configs)
     list(APPEND Caffe_DEFINITIONS -DCPU_ONLY)
   endif()
 
+  if(USE_OPENCV)
+    list(APPEND Caffe_DEFINITIONS -DUSE_OPENCV)
+  endif()
+
+  if(USE_LMDB)
+    list(APPEND Caffe_DEFINITIONS -DUSE_LMDB)
+  endif()
+
+  if(USE_LEVELDB)
+    list(APPEND Caffe_DEFINITIONS -DUSE_LEVELDB)
+  endif()
+
   if(NOT HAVE_CUDNN)
     set(HAVE_CUDNN FALSE)
   else()

cmake/Dependencies.cmake

@@ -29,19 +29,27 @@ include_directories(SYSTEM ${HDF5_INCLUDE_DIRS} ${HDF5_HL_INCLUDE_DIR})
 list(APPEND Caffe_LINKER_LIBS ${HDF5_LIBRARIES})
 
 # ---[ LMDB
-find_package(LMDB REQUIRED)
-include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
-list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
+if(USE_LMDB)
+  find_package(LMDB REQUIRED)
+  include_directories(SYSTEM ${LMDB_INCLUDE_DIR})
+  list(APPEND Caffe_LINKER_LIBS ${LMDB_LIBRARIES})
+  add_definitions(-DUSE_LMDB)
+endif()
 
 # ---[ LevelDB
-find_package(LevelDB REQUIRED)
-include_directories(SYSTEM ${LevelDB_INCLUDE})
-list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES})
+if(USE_LEVELDB)
+  find_package(LevelDB REQUIRED)
+  include_directories(SYSTEM ${LevelDB_INCLUDE})
+  list(APPEND Caffe_LINKER_LIBS ${LevelDB_LIBRARIES})
+  add_definitions(-DUSE_LEVELDB)
+endif()
 
 # ---[ Snappy
-find_package(Snappy REQUIRED)
-include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
-list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES})
+if(USE_LEVELDB)
+  find_package(Snappy REQUIRED)
+  include_directories(SYSTEM ${Snappy_INCLUDE_DIR})
+  list(APPEND Caffe_LINKER_LIBS ${Snappy_LIBRARIES})
+endif()
 
 # ---[ CUDA
 include(cmake/Cuda.cmake)
@@ -57,13 +65,16 @@ if(NOT HAVE_CUDA)
 endif()
 
 # ---[ OpenCV
-find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
-if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
-  find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
+if(USE_OPENCV)
+  find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
+  if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
+    find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
+  endif()
+  include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
+  list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
+  message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
+  add_definitions(-DUSE_OPENCV)
 endif()
-include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
-list(APPEND Caffe_LINKER_LIBS ${OpenCV_LIBS})
-message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
 
 # ---[ BLAS
 if(NOT APPLE)

cmake/Summary.cmake

@@ -114,17 +114,26 @@ function(caffe_print_configuration_summary)
   caffe_status("  BUILD_matlab      :   ${BUILD_matlab}")
   caffe_status("  BUILD_docs        :   ${BUILD_docs}")
   caffe_status("  CPU_ONLY          :   ${CPU_ONLY}")
+  caffe_status("  USE_LMDB          :   ${USE_LMDB}")
+  caffe_status("  USE_LEVELDB       :   ${USE_LEVELDB}")
+  caffe_status("  USE_OPENCV        :   ${USE_OPENCV}")
   caffe_status("")
   caffe_status("Dependencies:")
   caffe_status("  BLAS              : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")
   caffe_status("  Boost             :   Yes (ver. ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION})")
   caffe_status("  glog              :   Yes")
   caffe_status("  gflags            :   Yes")
   caffe_status("  protobuf          : " PROTOBUF_FOUND THEN "Yes (ver. ${PROTOBUF_VERSION})" ELSE "No" )
-  caffe_status("  lmdb              : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No")
-  caffe_status("  Snappy            : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" )
-  caffe_status("  LevelDB           : " LEVELDB_FOUND THEN  "Yes (ver. ${LEVELDB_VERSION})" ELSE "No")
-  caffe_status("  OpenCV            :   Yes (ver. ${OpenCV_VERSION})")
+  if(USE_LMDB)
+    caffe_status("  lmdb              : " LMDB_FOUND THEN "Yes (ver. ${LMDB_VERSION})" ELSE "No")
+  endif()
+  if(USE_LEVELDB)
+    caffe_status("  LevelDB           : " LEVELDB_FOUND THEN  "Yes (ver. ${LEVELDB_VERSION})" ELSE "No")
+    caffe_status("  Snappy            : " SNAPPY_FOUND THEN "Yes (ver. ${Snappy_VERSION})" ELSE "No" )
+  endif()
+  if(USE_OPENCV)
+    caffe_status("  OpenCV            :   Yes (ver. ${OpenCV_VERSION})")
+  endif()
   caffe_status("  CUDA              : " HAVE_CUDA THEN "Yes (ver. ${CUDA_VERSION})" ELSE "No" )
   caffe_status("")
   if(HAVE_CUDA)
@@ -165,4 +174,3 @@ function(caffe_print_configuration_summary)
   caffe_status("  Install path      :   ${CMAKE_INSTALL_PREFIX}")
   caffe_status("")
 endfunction()
-

cmake/Templates/CaffeConfig.cmake.in

@@ -17,22 +17,24 @@
 #   Caffe_HAVE_CUDNN   - signals about cuDNN support
 
 
-# OpenCV dependency
+# OpenCV dependency (optional)
 
-if(NOT OpenCV_FOUND)
-  set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@")
-  if(Caffe_OpenCV_CONFIG_PATH)
-    get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE)
+if(@USE_OPENCV@)
+  if(NOT OpenCV_FOUND)
+    set(Caffe_OpenCV_CONFIG_PATH "@OpenCV_CONFIG_PATH@")
+    if(Caffe_OpenCV_CONFIG_PATH)
+      get_filename_component(Caffe_OpenCV_CONFIG_PATH ${Caffe_OpenCV_CONFIG_PATH} ABSOLUTE)
 
-    if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core)
-      message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}")
-      include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake)
-    endif()
+      if(EXISTS ${Caffe_OpenCV_CONFIG_PATH} AND NOT TARGET opencv_core)
+        message(STATUS "Caffe: using OpenCV config from ${Caffe_OpenCV_CONFIG_PATH}")
+        include(${Caffe_OpenCV_CONFIG_PATH}/OpenCVModules.cmake)
+      endif()
 
-  else()
-    find_package(OpenCV REQUIRED)
+    else()
+      find_package(OpenCV REQUIRED)
+    endif()
+    unset(Caffe_OpenCV_CONFIG_PATH)
   endif()
-  unset(Caffe_OpenCV_CONFIG_PATH)
 endif()
 
 # Compute paths

cmake/Templates/caffe_config.h.in

@@ -30,3 +30,8 @@
 
 /* Matlab */
 #cmakedefine HAVE_MATLAB
+
+/* IO libraries */
+#cmakedefine USE_OPENCV
+#cmakedefine USE_LMDB
+#cmakedefine USE_LEVELDB

docs/installation.md

@@ -17,16 +17,19 @@ When updating Caffe, it's best to `make clean` before re-compiling.
 
 ## Prerequisites
 
-Caffe has several dependencies.
+Caffe has several dependencies:
 
 * [CUDA](https://developer.nvidia.com/cuda-zone) is required for GPU mode.
     * library version 7.0 and the latest driver version are recommended, but 6.* is fine too
     * 5.5, and 5.0 are compatible but considered legacy
 * [BLAS](http://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) via ATLAS, MKL, or OpenBLAS.
 * [Boost](http://www.boost.org/) >= 1.55
+* `protobuf`, `glog`, `gflags`, `hdf5`
+
+Optional dependencies:
+
 * [OpenCV](http://opencv.org/) >= 2.4 including 3.0
-* `protobuf`, `glog`, `gflags`
-* IO libraries `hdf5`, `leveldb`, `snappy`, `lmdb`
+* IO libraries: `lmdb`, `leveldb` (note: leveldb requires `snappy`)
 
 Pycaffe and Matcaffe interfaces have their own natural needs.
 

examples/cpp_classification/classification.cpp

@@ -1,14 +1,17 @@
 #include <caffe/caffe.hpp>
+#ifdef USE_OPENCV
 #include <opencv2/core/core.hpp>
 #include <opencv2/highgui/highgui.hpp>
 #include <opencv2/imgproc/imgproc.hpp>
+#endif  // USE_OPENCV
 #include <algorithm>
 #include <iosfwd>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
+#ifdef USE_OPENCV
 using namespace caffe;  // NOLINT(build/namespaces)
 using std::string;
 
@@ -255,3 +258,8 @@ int main(int argc, char** argv) {
               << p.first << "\"" << std::endl;
   }
 }
+#else
+int main(int argc, char** argv) {
+  LOG(FATAL) << "This example requires OpenCV; compile with USE_OPENCV.";
+}
+#endif  // USE_OPENCV

examples/mnist/convert_mnist_data.cpp

@@ -9,9 +9,13 @@
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 #include <google/protobuf/text_format.h>
+
+#if defined(USE_LEVELDB) && defined(USE_LMDB)
 #include <leveldb/db.h>
 #include <leveldb/write_batch.h>
 #include <lmdb.h>
+#endif
+
 #include <stdint.h>
 #include <sys/stat.h>
 
@@ -20,6 +24,8 @@
 
 #include "caffe/proto/caffe.pb.h"
 
+#if defined(USE_LEVELDB) && defined(USE_LMDB)
+
 using namespace caffe;  // NOLINT(build/namespaces)
 using std::string;
 
@@ -196,3 +202,9 @@ int main(int argc, char** argv) {
   }
   return 0;
 }
+#else
+int main(int argc, char** argv) {
+  LOG(FATAL) << "This example requires LevelDB and LMDB; " <<
+  "compile with USE_LEVELDB and USE_LMDB.";
+}
+#endif  // USE_LEVELDB and USE_LMDB

examples/siamese/convert_mnist_siamese_data.cpp

@@ -10,12 +10,14 @@
 
 #include "glog/logging.h"
 #include "google/protobuf/text_format.h"
-#include "leveldb/db.h"
 #include "stdint.h"
 
 #include "caffe/proto/caffe.pb.h"
 #include "caffe/util/math_functions.hpp"
 
+#ifdef USE_LEVELDB
+#include "leveldb/db.h"
+
 uint32_t swap_endian(uint32_t val) {
     val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
     return (val << 16) | (val >> 16);
@@ -121,3 +123,8 @@ int main(int argc, char** argv) {
   }
   return 0;
 }
+#else
+int main(int argc, char** argv) {
+  LOG(FATAL) << "This example requires LevelDB; compile with USE_LEVELDB.";
+}
+#endif  // USE_LEVELDB

include/caffe/data_layers.hpp

@@ -4,7 +4,6 @@
 #include <string>
 #include <utility>
 #include <vector>
-
 #include "hdf5.h"
 
 #include "caffe/blob.hpp"
@@ -275,8 +274,10 @@ class MemoryDataLayer : public BaseDataLayer<Dtype> {
   virtual inline int ExactNumTopBlobs() const { return 2; }
 
   virtual void AddDatumVector(const vector<Datum>& datum_vector);
+#ifdef USE_OPENCV
   virtual void AddMatVector(const vector<cv::Mat>& mat_vector,
       const vector<int>& labels);
+#endif  // USE_OPENCV
 
   // Reset should accept const pointers, but can't, because the memory
   //  will be given to Blob, which is mutable