From 7a4b26fc5aa1823f3fbcf319cf60e1295a70d4e7 Mon Sep 17 00:00:00 2001 From: rmdg88 Date: Fri, 4 Oct 2024 21:20:28 +0100 Subject: [PATCH] optional system deps --- .github/actions/setup-poetry/action.yml | 9 +- .github/scripts/build_rhel.sh | 36 ++++++++ .github/workflows/checks.yml | 5 +- .github/workflows/ci.yml | 3 +- .github/workflows/rhel.yml | 12 +++ .github/workflows/wheels.yml | 80 ++++++++++++++--- .gitignore | 94 -------------------- CMakeLists.txt | 109 ++++++++++++------------ build.py | 76 +++++++++++------ cmake/extlib_cxxopts.cmake | 49 ++++++----- cmake/extlib_fasttext.cmake | 58 +++++++------ cmake/extlib_fmt.cmake | 65 ++++++++------ cmake/extlib_json.cmake | 48 +++++++---- cmake/extlib_loguru.cmake | 55 +++++++----- cmake/extlib_pcre2.cmake | 67 ++++++++------- cmake/extlib_sentencepiece.cmake | 102 +++++++++++----------- cmake/extlib_utf8.git.cmake | 47 ++++++---- cmake/os_opts.cmake | 26 +++--- poetry.lock | 24 +++--- pyproject.toml | 13 ++- src/libraries.h | 2 +- 21 files changed, 556 insertions(+), 424 deletions(-) create mode 100644 .github/scripts/build_rhel.sh create mode 100644 .github/workflows/rhel.yml delete mode 100644 .gitignore diff --git a/.github/actions/setup-poetry/action.yml b/.github/actions/setup-poetry/action.yml index aae7453f..77a1324e 100644 --- a/.github/actions/setup-poetry/action.yml +++ b/.github/actions/setup-poetry/action.yml @@ -10,10 +10,17 @@ runs: - name: Install poetry run: pipx install poetry==1.8.3 shell: bash - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 + id: py with: python-version: ${{ inputs.python-version }} + update-environment: false cache: 'poetry' + - name: Setup poetry env with correct python + run: | + poetry env use ${{ steps.py.outputs.python-path }} + poetry run python --version + shell: bash - name: Install only dependencies and not the package itself run: poetry install --all-extras --no-root shell: bash diff --git a/.github/scripts/build_rhel.sh b/.github/scripts/build_rhel.sh new file mode 100644 index 00000000..e136293c --- /dev/null +++ b/.github/scripts/build_rhel.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e # trigger failure on error - do not remove! +set -x # display command on output + +# Build the Python package with Poetry +poetry build -f sdist + +# Docker build +docker build -f - . < SUCCESS") return True - print(f" -> ERROR with message: '{message}'\n") + print(f" -> ERROR with message: '{message}'\n") return False - + + def build_local(num_threads: int): - - if not os.path.exists(BUILD_DIR): - print("python executable: ", sys.executable) - - cmd = f"cmake -B {BUILD_DIR} -DPYTHON_EXECUTABLE={sys.executable}" - run(cmd, cwd=ROOT_DIR) - else: - print(f"build directory detected: {BUILD_DIR}") - - cmd = f"cmake --build {BUILD_DIR} --target install" + + USE_SYSTEM_DEPS = os.getenv("USE_SYSTEM_DEPS", "OFF") + + + print("python prefix: ", sys.exec_prefix) + print("python executable: ", sys.executable) + config_cmd = [ + "cmake", + "-B", f"{BUILD_DIR}", + f"-DUSE_SYSTEM_DEPS={USE_SYSTEM_DEPS}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + ] + config_cmd.extend(get_pybind11_cmake_args()) + success = run(config_cmd, cwd=ROOT_DIR) + if not success: + raise RuntimeError("Error building.") + + build_cmd = [ + "cmake", + "--build", f"{BUILD_DIR}", + "--target=install", + ] if num_threads > 1: - cmd += f" -j {num_threads}" - run(cmd, cwd=ROOT_DIR) + build_cmd.extend(["-j", f"{num_threads}"]) + success = run(build_cmd, cwd=ROOT_DIR) + if not success: + raise RuntimeError("Error building.") if "__main__" == __name__: diff --git a/cmake/extlib_cxxopts.cmake b/cmake/extlib_cxxopts.cmake index 1f625b6d..3ad700c9 100644 --- a/cmake/extlib_cxxopts.cmake +++ b/cmake/extlib_cxxopts.cmake @@ -1,33 +1,42 @@ -cmake_minimum_required (VERSION 3.5) message(STATUS "entering in extlib_cxxopts.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "cxxopts") -set(CXXOPTS_TAG v2.2.0) -set(CXXOPTS_URL https://github.com/jarro2783/cxxopts.git) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libcxxopts REQUIRED IMPORTED_TARGET cxxopts) -ExternalProject_Add(extlib_cxxopts - PREFIX extlib_cxxopts + add_library(${ext_name} ALIAS PkgConfig::libcxxopts) +else() + include(ExternalProject) + include(CMakeParseArguments) - GIT_TAG ${CXXOPTS_TAG} - GIT_REPOSITORY ${CXXOPTS_URL} + set(CXXOPTS_URL https://github.com/jarro2783/cxxopts.git) + set(CXXOPTS_TAG v3.2.0) - INSTALL_DIR ${CXXOPTS_PREFIX_INSTALL_DIR} + ExternalProject_Add(extlib_cxxopts - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" + PREFIX extlib_cxxopts - BUILD_COMMAND "" - BUILD_ALWAYS OFF + UPDATE_COMMAND "" + GIT_REPOSITORY ${CXXOPTS_URL} + GIT_TAG ${CXXOPTS_TAG} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory /include/ ${EXTERNALS_PREFIX_PATH}/include/ + BUILD_ALWAYS OFF - LOG_DOWNLOAD ON - LOG_BUILD ON + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + + CMAKE_ARGS \\ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \\ + -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} + + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON ) -add_library(cxxopts INTERFACE) -add_custom_target(install_extlib_cxxopts DEPENDS extlib_cxxopts) -add_dependencies(cxxopts install_extlib_cxxopts) \ No newline at end of file + add_library(${ext_name} INTERFACE) + add_dependencies(${ext_name} extlib_cxxopts) + set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include + ) +endif() diff --git a/cmake/extlib_fasttext.cmake b/cmake/extlib_fasttext.cmake index 1affde77..2bbd8c1b 100644 --- a/cmake/extlib_fasttext.cmake +++ b/cmake/extlib_fasttext.cmake @@ -1,36 +1,46 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_fasttext.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "fasttext") -set(FASTTEXT_URL https://github.com/PeterStaar-IBM/fastText.git) -set(FASTTEXT_TAG 9d5b2a2b364f49ed2707ff3be48a0f1ba6d86022) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libfasttext_pic REQUIRED IMPORTED_TARGET fasttext) -set(CXX_FLAGS "${ENV_ARCHFLAGS} -O3") + add_library(${ext_name} ALIAS PkgConfig::libfasttext_pic) +else() + include(ExternalProject) + include(CMakeParseArguments) -ExternalProject_Add(extlib_fasttext - PREFIX extlib_fasttext + set(FASTTEXT_URL https://github.com/PeterStaar-IBM/fastText.git) + set(FASTTEXT_TAG 9d5b2a2b364f49ed2707ff3be48a0f1ba6d86022) - GIT_REPOSITORY ${FASTTEXT_URL} - GIT_TAG ${FASTTEXT_TAG} + ExternalProject_Add(extlib_fasttext - UPDATE_COMMAND "" + PREFIX extlib_fasttext - CMAKE_ARGS \\ - -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ - -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ - -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\ - -DCMAKE_INSTALL_BINDIR=${EXTERNALS_PREFIX_PATH}/bin \\ - -DCMAKE_INSTALL_INCLUDEDIR=${EXTERNALS_PREFIX_PATH}/include + GIT_REPOSITORY ${FASTTEXT_URL} + GIT_TAG ${FASTTEXT_TAG} - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + UPDATE_COMMAND "" - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - LOG_BUILD ON) + BUILD_ALWAYS OFF + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} -add_library(fasttext STATIC IMPORTED) -set_target_properties(fasttext PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfasttext_pic.a) -add_dependencies(fasttext extlib_fasttext) + CMAKE_ARGS \\ + -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ + -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ + -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\ + -DCMAKE_INSTALL_BINDIR=${EXTERNALS_PREFIX_PATH}/bin \\ + -DCMAKE_INSTALL_INCLUDEDIR=${EXTERNALS_PREFIX_PATH}/include + + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_BUILD ON + ) + + add_library(${ext_name} STATIC IMPORTED) + add_dependencies(${ext_name} extlib_fasttext) + set_target_properties(${ext_name} PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfasttext_pic.a INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include + ) +endif() diff --git a/cmake/extlib_fmt.cmake b/cmake/extlib_fmt.cmake index da3b8970..075a594a 100644 --- a/cmake/extlib_fmt.cmake +++ b/cmake/extlib_fmt.cmake @@ -1,40 +1,49 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_fmt.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "fmt") -set(FMT_URL https://github.com/fmtlib/fmt.git) -set(FMT_TAG 10.2.1) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libfmt REQUIRED IMPORTED_TARGET fmt) -message(STATUS "extlib_fmt cxx-flags: " ${CMAKE_CXX_FLAGS}) + add_library(${ext_name} ALIAS PkgConfig::libfmt) +else() + include(ExternalProject) + include(CMakeParseArguments) -ExternalProject_Add(extlib_fmt - PREFIX extlib_fmt + set(FMT_URL https://github.com/fmtlib/fmt.git) + set(FMT_TAG 10.2.1) - GIT_REPOSITORY ${FMT_URL} - GIT_TAG ${FMT_TAG} + message(STATUS "extlib_fmt cxx-flags: " ${CMAKE_CXX_FLAGS}) - UPDATE_COMMAND "" - CMAKE_ARGS \\ - -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ - -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ - -DFMT_LIB_DIR=lib \\ - -DFMT_TEST=OFF \\ - -DFMT_DOC=OFF \\ - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE + ExternalProject_Add(extlib_fmt - #-DCMAKE_CXX_FLAGS=${ENV_ARCHFLAGS};-O3 \\ - #-DCMAKE_CXX_FLAGS=-O3 \\ - #-DFMT_INSTALL=OFF + PREFIX extlib_fmt - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + GIT_REPOSITORY ${FMT_URL} + GIT_TAG ${FMT_TAG} - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - LOG_BUILD ON) + UPDATE_COMMAND "" -add_library(fmt STATIC IMPORTED) -set_target_properties(fmt PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfmt.a) -add_dependencies(fmt extlib_fmt) + BUILD_ALWAYS OFF + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + + CMAKE_ARGS \\ + -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ + -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ + -DFMT_LIB_DIR=lib \\ + -DFMT_TEST=OFF \\ + -DFMT_DOC=OFF \\ + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE + + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_BUILD ON + ) + + add_library(${ext_name} STATIC IMPORTED) + add_dependencies(${ext_name} extlib_fmt) + set_target_properties(${ext_name} PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfmt.a INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include + ) +endif() diff --git a/cmake/extlib_json.cmake b/cmake/extlib_json.cmake index c5b4592d..41ffc763 100644 --- a/cmake/extlib_json.cmake +++ b/cmake/extlib_json.cmake @@ -1,29 +1,41 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_json.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "json") -set(JSON_URL https://github.com/nlohmann/json.git) -set(JSON_TAG v3.11.3) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libjson REQUIRED IMPORTED_TARGET nlohmann_json) -ExternalProject_Add(extlib_json - PREFIX extlib_json + add_library(${ext_name} ALIAS PkgConfig::libjson) +else() + include(ExternalProject) + include(CMakeParseArguments) - GIT_REPOSITORY ${JSON_URL} - GIT_TAG ${JSON_TAG} + set(JSON_URL https://github.com/nlohmann/json.git) + set(JSON_TAG v3.11.3) - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" + ExternalProject_Add(extlib_json - BUILD_COMMAND "" - BUILD_ALWAYS OFF + PREFIX extlib_json - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory /include/ ${EXTERNALS_PREFIX_PATH}/include/ + GIT_REPOSITORY ${JSON_URL} + GIT_TAG ${JSON_TAG} + + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_ALWAYS OFF + + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory /include/ ${EXTERNALS_PREFIX_PATH}/include/ + + LOG_DOWNLOAD ON + LOG_BUILD ON ) -add_library(json INTERFACE) -add_custom_target(install_extlib_json DEPENDS extlib_json) -add_dependencies(json install_extlib_json) \ No newline at end of file + add_library(${ext_name} INTERFACE) + add_dependencies(${ext_name} extlib_json) + set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include + ) +endif() diff --git a/cmake/extlib_loguru.cmake b/cmake/extlib_loguru.cmake index a7084123..9ce1a700 100644 --- a/cmake/extlib_loguru.cmake +++ b/cmake/extlib_loguru.cmake @@ -1,32 +1,47 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_loguru.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "loguru") -set(LOGURU_INCLUDE_DIR ${EXTERNALS_PREFIX_PATH}/include/loguru) -execute_process(COMMAND mkdir -p ${LOGURU_INCLUDE_DIR}) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libloguru REQUIRED IMPORTED_TARGET loguru) -set(LOGURU_URL https://github.com/emilk/loguru) -set(LOGURU_TAG v2.1.0) + add_library(${ext_name} ALIAS PkgConfig::libloguru) +else() + include(ExternalProject) + include(CMakeParseArguments) -ExternalProject_Add(extlib_loguru - PREFIX extlib_loguru + set(LOGURU_URL https://github.com/emilk/loguru) + set(LOGURU_TAG v2.1.0) - GIT_REPOSITORY ${LOGURU_URL} - GIT_TAG ${LOGURU_TAG} + set(LOGURU_INCLUDE_DIR ${EXTERNALS_PREFIX_PATH}/include/loguru) - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" + # Ensure the directory for headers exists + execute_process(COMMAND mkdir -p ${LOGURU_INCLUDE_DIR}) - BUILD_COMMAND "" - BUILD_ALWAYS OFF + ExternalProject_Add(extlib_loguru - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy /loguru.hpp ${LOGURU_INCLUDE_DIR} /loguru.cpp ${LOGURU_INCLUDE_DIR} + PREFIX extlib_loguru + + GIT_REPOSITORY ${LOGURU_URL} + GIT_TAG ${LOGURU_TAG} + + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + BUILD_ALWAYS OFF + + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy /loguru.hpp ${LOGURU_INCLUDE_DIR} && + ${CMAKE_COMMAND} -E copy /loguru.cpp ${LOGURU_INCLUDE_DIR} + + LOG_DOWNLOAD ON + LOG_BUILD ON ) -add_library(loguru INTERFACE) -add_custom_target(install_extlib_loguru DEPENDS extlib_loguru) -add_dependencies(loguru install_extlib_loguru) \ No newline at end of file + add_library(${ext_name} INTERFACE) + add_dependencies(${ext_name} extlib_loguru) + set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${LOGURU_INCLUDE_DIR} + ) +endif() diff --git a/cmake/extlib_pcre2.cmake b/cmake/extlib_pcre2.cmake index 48b03548..6c1a817b 100644 --- a/cmake/extlib_pcre2.cmake +++ b/cmake/extlib_pcre2.cmake @@ -1,43 +1,48 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_pcre2.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "pcre2") -set(PCRE2_URL https://github.com/PCRE2Project/pcre2.git) -#set(PCRE2_TAG pcre2-10.44) +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libpcre2-8 REQUIRED IMPORTED_TARGET libpcre2-8) -ExternalProject_Add(extlib_pcre2 - PREFIX extlib_pcre2 + add_library(${ext_name} ALIAS PkgConfig::libpcre2-8) +else() + include(ExternalProject) + include(CMakeParseArguments) - GIT_REPOSITORY ${PCRE2_URL} - #GIT_TAG ${PCRE2_TAG} + set(PCRE2_URL https://github.com/PCRE2Project/pcre2.git) + #set(PCRE2_TAG pcre2-10.44) - UPDATE_COMMAND "" - - CMAKE_ARGS \\ - -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ - -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\ - -DCMAKE_OSX_ARCHITECTURES=${ENV_ARCH} \\ - -DBUILD_SHARED_LIBS=OFF \\ - -DBUILD_STATIC_LIBS=ON \\ - -DPCRE2_STATIC_PIC=ON \\ - -DPCRE2_SHOW_REPORT=OFF - #-DPCRE2_BUILD_PCRE2GREP=OFF + ExternalProject_Add(extlib_pcre2 - #-DCMAKE_CXX_FLAGS="-O3 ${ENV_ARCHFLAGS}" \\ + PREFIX extlib_pcre2 - BUILD_ALWAYS OFF + GIT_REPOSITORY ${PCRE2_URL} + #GIT_TAG ${PCRE2_TAG} - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} - #LIBRARY_DIR ${EXTERNALS_PREFIX_PATH}/lib + UPDATE_COMMAND "" - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - LOG_BUILD ON) + CMAKE_ARGS \\ + -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ + -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\ + -DCMAKE_OSX_ARCHITECTURES=${ENV_ARCH} \\ + -DBUILD_SHARED_LIBS=OFF \\ + -DBUILD_STATIC_LIBS=ON \\ + -DPCRE2_STATIC_PIC=ON \\ + -DPCRE2_SHOW_REPORT=OFF -add_library(pcre2 STATIC IMPORTED) -set_target_properties(pcre2 PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libpcre2-8.a) -#set_target_properties(pcre2 PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib64/libpcre2-8.a) -add_dependencies(pcre2 extlib_pcre2) + BUILD_ALWAYS OFF + + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_BUILD ON + ) + + add_library(${ext_name} STATIC IMPORTED) + add_dependencies(${ext_name} extlib_pcre2) + set_target_properties(${ext_name} PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libpcre2-8.a INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include + ) +endif() diff --git a/cmake/extlib_sentencepiece.cmake b/cmake/extlib_sentencepiece.cmake index 6bbd6fef..da372e39 100644 --- a/cmake/extlib_sentencepiece.cmake +++ b/cmake/extlib_sentencepiece.cmake @@ -1,51 +1,57 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_sentencepiece.cmake") -include(ExternalProject) -include(CMakeParseArguments) - -set(SENTENCEPIECE_URL https://github.com/google/sentencepiece.git) -set(SENTENCEPIECE_TAG v0.2.0) - -message(STATUS "extlib_sentencepiece cxx-flags: " ${CMAKE_CXX_FLAGS}) - -ExternalProject_Add(extlib_sentencepiece - PREFIX extlib_sentencepiece - - GIT_REPOSITORY ${SENTENCEPIECE_URL} - GIT_TAG ${SENTENCEPIECE_TAG} - - UPDATE_COMMAND "" - CMAKE_ARGS \\ - -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ - -DCMAKE_INSTALL_LIBDIR=lib \\ - -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ - -DSPM_BUILD_TEST=OFF \\ - -DSPM_COVERAGE=OFF \\ - -DSPM_ENABLE_NFKC_COMPILE=OFF \\ - -DSPM_ENABLE_SHARED=OFF \\ - -DSPM_ENABLE_TCMALLOC=OFF \\ - -DSPM_ENABLE_TENSORFLOW_SHARED=OFF \\ - -DSPM_NO_THREADLOCAL=OFF \\ - -DSPM_TCMALLOC_STATIC=OFF \\ - -DSPM_USE_BUILTIN_PROTOBUF=ON \\ - -DSPM_USE_EXTERNAL_ABSL=OFF \\ - -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE - - #-DCMAKE_CXX_FLAGS=${ENV_ARCHFLAGS} \\ - - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} - - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - # LOG_BUILD ON -) - -add_library(sentencepiece STATIC IMPORTED) -set_target_properties(sentencepiece PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libsentencepiece.a) -add_dependencies(sentencepiece extlib_sentencepiece) - -add_library(sentencepiece_train STATIC IMPORTED) -set_target_properties(sentencepiece_train PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libsentencepiece_train.a) -add_dependencies(sentencepiece_train extlib_sentencepiece) +set(ext_name "sentencepiece") + +if(USE_SYSTEM_DEPS) + find_package(PkgConfig) + pkg_check_modules(libsentencepiece REQUIRED IMPORTED_TARGET sentencepiece) + + add_library(${ext_name} ALIAS PkgConfig::libsentencepiece) +else() + include(ExternalProject) + include(CMakeParseArguments) + + set(SENTENCEPIECE_URL https://github.com/google/sentencepiece.git) + set(SENTENCEPIECE_TAG v0.2.0) + + message(STATUS "extlib_sentencepiece cxx-flags: " ${CMAKE_CXX_FLAGS}) + + ExternalProject_Add(extlib_sentencepiece + PREFIX extlib_sentencepiece + + GIT_REPOSITORY ${SENTENCEPIECE_URL} + GIT_TAG ${SENTENCEPIECE_TAG} + + UPDATE_COMMAND "" + CMAKE_ARGS \\ + -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\ + -DCMAKE_INSTALL_LIBDIR=lib \\ + -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\ + -DSPM_BUILD_TEST=OFF \\ + -DSPM_COVERAGE=OFF \\ + -DSPM_ENABLE_NFKC_COMPILE=OFF \\ + -DSPM_ENABLE_SHARED=OFF \\ + -DSPM_ENABLE_TCMALLOC=OFF \\ + -DSPM_ENABLE_TENSORFLOW_SHARED=OFF \\ + -DSPM_NO_THREADLOCAL=OFF \\ + -DSPM_TCMALLOC_STATIC=OFF \\ + -DSPM_USE_BUILTIN_PROTOBUF=ON \\ + -DSPM_USE_EXTERNAL_ABSL=OFF \\ + -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE + + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_BUILD ON + ) + + add_library(${ext_name} STATIC IMPORTED) + set_target_properties(${ext_name} PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libsentencepiece.a) + add_dependencies(${ext_name} extlib_sentencepiece) + + add_library(${ext_name}_train STATIC IMPORTED) + set_target_properties(${ext_name}_train PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libsentencepiece_train.a) + add_dependencies(${ext_name}_train extlib_sentencepiece) +endif() diff --git a/cmake/extlib_utf8.git.cmake b/cmake/extlib_utf8.git.cmake index ae06a458..e369dc60 100644 --- a/cmake/extlib_utf8.git.cmake +++ b/cmake/extlib_utf8.git.cmake @@ -1,28 +1,41 @@ -cmake_minimum_required(VERSION 3.5) message(STATUS "entering in extlib_utf8.cmake") -include(ExternalProject) -include(CMakeParseArguments) +set(ext_name "utf8") -set(UTF8_URL https://github.com/nemtrif/utfcpp.git) +if(USE_SYSTEM_DEPS) + # this will define the utf8cpp target + find_package(utf8cpp REQUIRED) -ExternalProject_Add(extlib_utf8 - PREFIX extlib_utf8 + add_library(${ext_name} INTERFACE IMPORTED) + add_dependencies(${ext_name} utf8cpp) - GIT_REPOSITORY ${UTF8_URL} - #GIT_TAG ${UTF8_TAG} +else() - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" + include(ExternalProject) + include(CMakeParseArguments) - BUILD_COMMAND "" - BUILD_ALWAYS OFF + set(UTF8_URL https://github.com/nemtrif/utfcpp.git) + set(UTF8_TAG v4.0.5) - INSTALL_DIR ${EXTERNALS_PREFIX_PATH} - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory /source ${EXTERNALS_PREFIX_PATH}/include/utf8 + ExternalProject_Add(extlib_utf8 + PREFIX extlib_utf8 + + GIT_REPOSITORY ${UTF8_URL} + GIT_TAG ${UTF8_TAG} + + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + + BUILD_COMMAND "" + BUILD_ALWAYS OFF + + INSTALL_DIR ${EXTERNALS_PREFIX_PATH} + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory /source ${EXTERNALS_PREFIX_PATH}/include ) -add_library(utf8 INTERFACE) -add_custom_target(install_extlib_utf8 DEPENDS extlib_utf8) -add_dependencies(utf8 install_extlib_utf8) \ No newline at end of file + add_library(${ext_name} INTERFACE IMPORTED) + add_dependencies(${ext_name} extlib_utf8) + set_target_properties(${ext_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${EXTERNALS_PREFIX_PATH}/include) + +endif() diff --git a/cmake/os_opts.cmake b/cmake/os_opts.cmake index 2d4ad4cb..995af492 100644 --- a/cmake/os_opts.cmake +++ b/cmake/os_opts.cmake @@ -1,38 +1,36 @@ -cmake_minimum_required (VERSION 3.5) message(STATUS "entering in 'os_opts.cmake'") if(WIN32) message(STATUS "compiling on windows") - set(LIB_LINK fmt json loguru cxxopts pcre2 sentencepiece sentencepiece_train fasttext utf8 z ws2_32 user32) - + set(LIB_LINK) + set(OS_DEPENDENCIES) + elseif(APPLE) message(STATUS "compiling on mac-osx") #set(CMAKE_MACOSX_RPATH 1) find_library(FoundationLib Foundation) + find_package(ZLIB) #message("LIB: ${FoundationLib}") find_library(SystemConfigurationLib SystemConfiguration) #message("LIB: ${SystemConfigurationLib}") - #set(LIB_LINK json json_schema loguru cxxopts pcre2 fasttext utf8 andromeda_pos ${OPENSSL_LIBRARIES} ldap z) - #set(LIB_LINK json loguru cxxopts pcre2 fasttext utf8 andromeda_pos ${OPENSSL_LIBRARIES} ldap z) - #set(LIB_LINK json loguru cxxopts pcre2 fasttext utf8 ${OPENSSL_LIBRARIES} ldap z) - set(LIB_LINK fmt json loguru cxxopts pcre2 sentencepiece sentencepiece_train fasttext utf8 ldap z) + # set(LIB_LINK qpdf jpeg utf8 z) + set(OS_DEPENDENCIES ZLIB::ZLIB) - list(APPEND LIB_LINK ${FoundationLib} ${SystemConfigurationLib}) + + set(LIB_LINK ${FoundationLib} ${SystemConfigurationLib}) elseif(UNIX) message(STATUS "compiling on linux") - #set(LIB_LINK json json_schema loguru cxxopts pcre2 fasttext utf8 andromeda_pos ${OPENSSL_LIBRARIES} z) - #set(LIB_LINK json loguru cxxopts pcre2 fasttext utf8 andromeda_pos ${OPENSSL_LIBRARIES} z) - #set(LIB_LINK json loguru cxxopts pcre2 fasttext utf8 ${OPENSSL_LIBRARIES} z) - set(LIB_LINK fmt json loguru cxxopts pcre2 sentencepiece sentencepiece_train fasttext utf8 z) - - list(APPEND LIB_LINK dl m pthread rt resolv) + # set(LIB_LINK qpdf jpeg utf8 z) + find_package(ZLIB) + set(LIB_LINK dl m pthread rt resolv) + set(OS_DEPENDENCIES ZLIB::ZLIB) endif() diff --git a/poetry.lock b/poetry.lock index c04f9015..85e70cff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -357,13 +357,13 @@ files = [ [[package]] name = "cibuildwheel" -version = "2.21.1" +version = "2.21.2" description = "Build Python wheels on CI with minimal configuration." optional = false python-versions = ">=3.8" files = [ - {file = "cibuildwheel-2.21.1-py3-none-any.whl", hash = "sha256:cd50ae4ef79cd095565558fda41d5056ae3edf1291a73245326d8a7a692f0c48"}, - {file = "cibuildwheel-2.21.1.tar.gz", hash = "sha256:b186b2723b928b9ca4d5d08d555f4f0fe0c3c69d5b80bb7fddd838dddc0c7f65"}, + {file = "cibuildwheel-2.21.2-py3-none-any.whl", hash = "sha256:60c3bd4e374c459dda09c4be5da5084e86e19310dbd9052f630b76bf3942aabb"}, + {file = "cibuildwheel-2.21.2.tar.gz", hash = "sha256:57b04c8afe1db6e03e78f9a4af8525410344a65a585a3369755433ce5fbc7754"}, ] [package.dependencies] @@ -2226,19 +2226,19 @@ idna2008 = ["idna"] [[package]] name = "rich" -version = "13.8.1" +version = "13.9.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"}, - {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"}, + {file = "rich-13.9.1-py3-none-any.whl", hash = "sha256:b340e739f30aa58921dc477b8adaa9ecdb7cecc217be01d93730ee1bc8aa83be"}, + {file = "rich-13.9.1.tar.gz", hash = "sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466"}, ] [package.dependencies] markdown-it-py = ">=2.2.0" pygments = ">=2.13.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] @@ -2448,13 +2448,13 @@ files = [ [[package]] name = "tomli" -version = "2.0.1" +version = "2.0.2" description = "A lil' TOML parser" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index e6bf94d5..4d0920a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,16 @@ packages = [{include = "deepsearch_glm"}] include = [ {path = "deepsearch_glm/*.so", format = "wheel"}, {path = "deepsearch_glm/*.pyd", format = "wheel"}, - {path = "deepsearch_glm/*.dll", format = "wheel"} + {path = "deepsearch_glm/*.dll", format = "wheel"}, + {path = "CMakeLists.txt", format = "sdist"}, + {path = "build.py", format = "sdist"}, + {path = "*.md", format = "sdist"}, + {path = "poetry.lock", format = "sdist"}, + {path = "app/*.cpp", format = "sdist"}, + {path = "cmake/", format = "sdist"}, + {path = "app/", format = "sdist"}, + {path = "src/", format = "sdist"}, + {path = "tests/", format = "sdist"}, ] build = "build.py" @@ -118,7 +127,7 @@ module = "matplotlib.*" ignore_missing_imports = true [build-system] -requires = ["poetry-core"] +requires = ["poetry-core", "pybind11>=2.13.1"] build-backend = "poetry.core.masonry.api" #[tool.poetry.build] diff --git a/src/libraries.h b/src/libraries.h index 39cadcf6..c0e835f6 100644 --- a/src/libraries.h +++ b/src/libraries.h @@ -20,7 +20,7 @@ #include //#include -#include +#include #include //#include #include