Skip to content

Commit

Permalink
feat(c++): recover the BUILD_ARROW_FROM_SOURCE option (#621)
Browse files Browse the repository at this point in the history
Signed-off-by: acezen <[email protected]>
  • Loading branch information
acezen authored Sep 10, 2024
1 parent c658cfa commit 241b644
Show file tree
Hide file tree
Showing 5 changed files with 248 additions and 89 deletions.
76 changes: 76 additions & 0 deletions .github/workflows/ci-nightly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: GraphAr C++ CI Nightly

on:
workflow_dispatch:
schedule:
# The notifications for scheduled workflows are sent to the user who
# last modified the cron syntax in the workflow file.
# Trigger the workflow at 03:00(CST) every day.
- cron: '00 19 * * *'
jobs:
GraphAr-ubuntu-arrow-from-source:
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'apache/incubator-graphar' }}
runs-on: ubuntu-latest
env:
GAR_TEST_DATA: ${{ github.workspace }}/graphar-testing/
steps:
- uses: actions/checkout@v4
with:
submodules: true

- name: Cache for ccache
uses: actions/cache@v3
with:
path: ~/.ccache
key: ${{ matrix.os }}-build-ccache-${{ hashFiles('**/git-modules.txt') }}
restore-keys: |
${{ matrix.os }}-build-ccache-
- name: Install dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev
# install Catch2 v3
git clone --branch v3.6.0 https://github.com/catchorg/Catch2.git --depth 1
pushd Catch2
cmake -Bbuild -H. -DBUILD_TESTING=OFF
sudo cmake --build build/ --target install
popd
git clone https://github.com/apache/incubator-graphar-testing.git $GAR_TEST_DATA --depth 1
- name: CMake
run: |
mkdir build
pushd build
cmake ../cpp -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DBUILD_EXAMPLES=ON -DBUILD_ARROW_FROM_SOURCE=ON
popd
- name: Build GraphAr
run: |
pushd build
make -j$(nproc)
make graphar-ccache-stats
popd
- name: Test
run: |
cd build
ctest --output-on-failure
232 changes: 148 additions & 84 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ option(ENABLE_DOCS "Enable documentation" OFF)
option(BUILD_DOCS_ONLY "Build docs only" OFF)
option(USE_STATIC_ARROW "Link arrow static library" OFF)
option(GRAPHAR_BUILD_STATIC "Build GraphAr as static libraries" OFF)
option(BUILD_ARROW_FROM_SOURCE "Build Arrow from source" OFF)

if (USE_STATIC_ARROW)
set(GRAPHAR_BUILD_STATIC ON)
Expand Down Expand Up @@ -176,59 +177,7 @@ macro(install_graphar_target target)
)
endmacro()

# ------------------------------------------------------------------------------
# building or find third party library
# ------------------------------------------------------------------------------
# check if arrow is installed
find_package(Arrow QUIET)
if (NOT ${Arrow_FOUND})
message(FATAL_ERROR "apache-arrow is required, please install it and retry")
endif()
find_package(ArrowDataset QUIET)
if (NOT ${ArrowDataset_FOUND})
message(FATAL_ERROR "apache-arrow-dataset is required, please install it and retry")
endif()
if (${Arrow_VERSION} VERSION_GREATER_EQUAL "12.0.0")
# ArrowAcero is available in Arrow 12.0.0 and later
find_package(ArrowAcero QUIET)
if (NOT ${ArrowAcero_FOUND})
message(FATAL_ERROR "apache-arrow-acero is required, please install it and retry")
endif()
endif()
# Check if ORC is enabled.
if (NOT ${ARROW_ORC})
message(WARNING "apache-arrow is built without ORC extension, ORC related functionalities will be disabled.")
else()
add_definitions(-DARROW_ORC) # Add macro, otherwise inconsistent in build phase on ubuntu.
endif()

find_package(Parquet QUIET)
if (NOT ${Parquet_FOUND})
message(FATAL_ERROR "parquet is required, please install it and retry")
endif()

find_package(Protobuf REQUIRED)
include_directories(${PROTOBUF_INCLUDE_DIRS})

macro(get_target_location var target)
if(TARGET ${target})
foreach(prop LOCATION LOCATION_NOCONFIG LOCATION_DEBUG LOCATION_RELEASE)
get_target_property(${var} ${target} ${prop})
if(NOT ("${${var}}" STREQUAL "${var}-NOTFOUND"))
break ()
endif()
endforeach()
endif()
endmacro()

include_directories(${CMAKE_CURRENT_BINARY_DIR}/src)
include_directories(src)

# ------------------------------------------------------------------------------
# generate graphar library
# ------------------------------------------------------------------------------
macro(build_graphar)

file(GLOB_RECURSE CORE_SRC_FILES "src/graphar/*.cc" ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/mini-yaml/yaml/*.cpp)
if(GRAPHAR_BUILD_STATIC)
add_library(graphar STATIC ${CORE_SRC_FILES})
Expand All @@ -237,9 +186,6 @@ macro(build_graphar)
endif()
install_graphar_target(graphar)
target_compile_features(graphar PRIVATE cxx_std_17)
target_include_directories(graphar PUBLIC $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
)
target_include_directories(graphar PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
target_link_libraries(graphar PRIVATE ${CMAKE_DL_LIBS})

Expand Down Expand Up @@ -272,7 +218,101 @@ macro(build_graphar)
endif()
endmacro()

build_graphar()
macro(build_graphar_with_arrow_bundled)
file(GLOB_RECURSE CORE_SRC_FILES "src/graphar/*.cc" ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/mini-yaml/yaml/*.cpp)
if(GRAPHAR_BUILD_STATIC)
add_library(graphar STATIC ${CORE_SRC_FILES})
else()
add_library(graphar SHARED ${CORE_SRC_FILES})
endif()
install_graphar_target(graphar)
target_compile_features(graphar PRIVATE cxx_std_17)
target_include_directories(graphar PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty)
target_include_directories(graphar SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR})
target_link_libraries(graphar PRIVATE ${CMAKE_DL_LIBS})

if(APPLE)
message(STATUS "Linking arrow bundled dependencies " ${GAR_PARQUET_STATIC_LIB} ${GAR_ACERO_STATIC_LIB})
target_link_libraries(graphar PRIVATE -Wl,-force_load gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_DATASET_STATIC_LIB}"
"${GAR_ARROW_ACERO_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}"
"-framework CoreFoundation"
"-framework Security")
else()
target_link_libraries(graphar PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_DATASET_STATIC_LIB}"
"${GAR_ARROW_ACERO_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive)
endif()

# if OpenSSL library exists, link the OpenSSL library.
# OpenSSL has to be linked after GAR_ARROW_BUNDLED_DEPS_STATIC_LIB
if(OPENSSL_FOUND)
target_link_libraries(graphar PUBLIC OpenSSL::SSL)
endif()
if (CURL_FOUND)
target_link_libraries(graphar PUBLIC ${CURL_LIBRARIES})
endif()
endmacro()

# ------------------------------------------------------------------------------
# building or find third party library
# ------------------------------------------------------------------------------

include_directories(${CMAKE_CURRENT_BINARY_DIR}/src)
include_directories(src)

if (BUILD_ARROW_FROM_SOURCE)
# the nessary dependencies for building arrow from source
find_package(OpenSSL REQUIRED)
if(OPENSSL_FOUND)
if(OPENSSL_VERSION LESS "1.1.0")
message(ERROR "The OpenSSL must be greater than or equal to 1.1.0, current version is ${OPENSSL_VERSION}")
endif()
endif()
find_package(CURL REQUIRED)

include(apache-arrow)
build_arrow()
add_definitions(-DARROW_ORC) # Add macro, otherwise inconsistent in build phase with not from source.
build_graphar_with_arrow_bundled()
else()
# check if arrow is installed
find_package(Arrow QUIET)
if (NOT ${Arrow_FOUND})
message(FATAL_ERROR "apache-arrow is required, please install it and retry")
endif()
find_package(ArrowDataset QUIET)
if (NOT ${ArrowDataset_FOUND})
message(FATAL_ERROR "apache-arrow-dataset is required, please install it and retry")
endif()
if (${Arrow_VERSION} VERSION_GREATER_EQUAL "12.0.0")
# ArrowAcero is available in Arrow 12.0.0 and later
find_package(ArrowAcero QUIET)
if (NOT ${ArrowAcero_FOUND})
message(FATAL_ERROR "apache-arrow-acero is required, please install it and retry")
endif()
endif()
# Check if ORC is enabled.
if (NOT ${ARROW_ORC})
message(WARNING "apache-arrow is built without ORC extension, ORC related functionalities will be disabled.")
else()
add_definitions(-DARROW_ORC) # Add macro, otherwise inconsistent in build phase on ubuntu.
endif()

find_package(Parquet QUIET)
if (NOT ${Parquet_FOUND})
message(FATAL_ERROR "parquet is required, please install it and retry")
endif()

find_package(Protobuf REQUIRED)
include_directories(${PROTOBUF_INCLUDE_DIRS})

build_graphar()
endif()

# ------------------------------------------------------------------------------
# build example
Expand All @@ -287,29 +327,40 @@ if (BUILD_EXAMPLES)
message(STATUS "Found example - " ${E_NAME})
add_executable(${E_NAME} examples/${E_NAME}.cc)
target_include_directories(${E_NAME} PRIVATE examples
${PROJECT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/Catch2/single_include>
)
target_include_directories(${E_NAME} SYSTEM PRIVATE ${Boost_INCLUDE_DIRS})
target_link_libraries(${E_NAME} PRIVATE graphar ${Boost_LIBRARIES} ${CMAKE_DL_LIBS})
if(APPLE)
if(USE_STATIC_ARROW)
target_link_libraries(${E_NAME} PRIVATE -Wl,-force_load
Arrow::arrow_static
Parquet::parquet_static)
if (BUILD_ARROW_FROM_SOURCE)
target_include_directories(${E_NAME} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR})
if (APPLE)
target_link_libraries(${E_NAME} PRIVATE -Wl,-force_load gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}")
else()
target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
target_link_libraries(${E_NAME} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive)
endif()
else()
if(USE_STATIC_ARROW)
target_link_libraries(${E_NAME} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive
Arrow::arrow_static
Parquet::parquet_static -Wl,--no-whole-archive)
if(APPLE)
if(USE_STATIC_ARROW)
target_link_libraries(${E_NAME} PRIVATE -Wl,-force_load
Arrow::arrow_static
Parquet::parquet_static)
else()
target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
endif()
else()
target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
if(USE_STATIC_ARROW)
target_link_libraries(${E_NAME} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive
Arrow::arrow_static
Parquet::parquet_static -Wl,--no-whole-archive)
else()
target_link_libraries(${E_NAME} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
endif()
endif()
endif()
endforeach()
Expand Down Expand Up @@ -363,23 +414,36 @@ if (BUILD_TESTS)
target_compile_features(${target} PRIVATE cxx_std_17)
target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/thirdparty)
target_link_libraries(${target} PRIVATE Catch2::Catch2WithMain graphar ${CMAKE_DL_LIBS})
if(APPLE)
if(USE_STATIC_ARROW)
target_link_libraries(${target} PRIVATE -Wl,-force_load
Arrow::arrow_static
Parquet::parquet_static)
if (BUILD_ARROW_FROM_SOURCE)
target_include_directories(${target} SYSTEM BEFORE PRIVATE ${GAR_ARROW_INCLUDE_DIR})
if (APPLE)
target_link_libraries(${target} PRIVATE -Wl,-force_load gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}")
else()
target_link_libraries(${target} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
target_link_libraries(${target} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static
"${GAR_PARQUET_STATIC_LIB}"
"${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive)
endif()
else()
if(USE_STATIC_ARROW)
target_link_libraries(${target} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive
Arrow::arrow_static
Parquet::parquet_static -Wl,--no-whole-archive)
if(APPLE)
if(USE_STATIC_ARROW)
target_link_libraries(${target} PRIVATE -Wl,-force_load
Arrow::arrow_static
Parquet::parquet_static)
else()
target_link_libraries(${target} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
endif()
else()
target_link_libraries(${target} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
if(USE_STATIC_ARROW)
target_link_libraries(${target} PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive
Arrow::arrow_static
Parquet::parquet_static -Wl,--no-whole-archive)
else()
target_link_libraries(${target} PRIVATE Arrow::arrow_shared
Parquet::parquet_shared)
endif()
endif()
endif()
target_include_directories(${target} PRIVATE ${PROJECT_SOURCE_DIR}/include $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/Catch2/single_include>)
Expand Down
10 changes: 10 additions & 0 deletions cpp/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,16 @@ Extra Build Options:
1. `-DGRAPHAR_BUILD_STATIC=ON`: Build GraphAr as static libraries.
2. `-DUSE_STATIC_ARROW=ON`: Link arrow static library to build GraphAr. If set this option, the option `GRAPHAR_BUILD_STATIC=ON` will be set.

### Building with Arrow from source
In case you want to build GraphAr as single static library including all dependencies, we include a [apache-arrow.cmake](cmake/apache-arrow.cmake) file that allows you to build Arrow and its dependencies from source and link it statically. To do this, you can follow the steps below:

```bash
mkdir build-static
cd build-static
cmake -DGRAPHAR_BUILD_STATIC=ON -DBUILD_ARROW_FROM_SOURCE=ON ..
make -j8 # if you have 8 CPU cores, otherwise adjust, use -j`nproc` for all cores
```

### Install

After the building, you can install the GraphAr C++ library with:
Expand Down
Loading

0 comments on commit 241b644

Please sign in to comment.