Merge pull request #70 from DS4SD/add-linux-arm-support

feat: add builds for linux arm64 architecture
DS4SD · Sep 9, 2024 · 7b99be7 · 7b99be7
2 parents 9ec1ec4 + b4bcb09
commit 7b99be7
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 17 deletions.
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -99,6 +99,7 @@ jobs:
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""  # do not run delocate-wheel before the re-tag
           CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
           ARCHFLAGS: -arch x86_64
+          BUILD_THREADS: "4"
         run: |
           echo "Building wheel ${CIBW_BUILD}"
           poetry run python --version
@@ -135,6 +136,7 @@ jobs:
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""  # do not run delocate-wheel before the re-tag
           CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
           ARCHFLAGS: -arch arm64
+          BUILD_THREADS: "4"
         run: |
           echo "Building wheel ${CIBW_BUILD}"
           poetry run python --version
@@ -168,15 +170,22 @@ jobs:
           ls -l ./dist
           poetry publish --no-interaction -vvv --username=__token__ --password=${{ secrets.PYPI_TOKEN }}
 
+      - name: Set up QEMU [linux]
+        if: matrix.os.name == 'ubuntu-latest'
+        uses: docker/setup-qemu-action@v3
+        with:
+            platforms: all
+
       - name: Build wheels [linux]
         if: matrix.os.name == 'ubuntu-latest'
         env:
-          CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
-          CIBW_ARCHS: x86_64
+          # CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
+          CIBW_ARCHS: auto x86_64 aarch64
           CIBW_PLATFORM: linux
-          CIBW_SKIP: "pp* *musllinux_* *_i686* *_s390*"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8"
+          CIBW_SKIP: "pp* *musllinux_* *_i686* *_s390* *pypy*"
+          CIBW_PROJECT_REQUIRES_PYTHON: "~=${{ matrix.python-version }}.0"
           CIBW_BUILD_VERBOSITY: 3
+          BUILD_THREADS: "8"
         run: |
           echo "Building wheel ${CIBW_BUILD}"
           poetry run python --version
@@ -185,7 +194,10 @@ jobs:
           cat ./pyproject.toml
           poetry run python -m cibuildwheel --output-dir ./wheelhouse
           ls -l ./wheelhouse
-          poetry run python -m zipfile --list ./wheelhouse/*.whl
+          for file in ./wheelhouse/*.whl; do
+            echo "Inspecting $file"
+            poetry run python -m zipfile --list "$file"
+          done
           mkdir ./dist
           cp wheelhouse/*.whl ./dist/
 

diff --git a/build.py b/build.py
@@ -30,7 +30,7 @@ def run(cmd, cwd="./"):
     print(f" -> ERROR with message: '{message}'\n")        
     return False
 
-def build_local(multi_threaded=True):
+def build_local(num_threads: int):
 
     if not os.path.exists(BUILD_DIR):
         print("python executable: ", sys.executable)
@@ -40,13 +40,12 @@ def build_local(multi_threaded=True):
     else:
         print(f"build directory detected: {BUILD_DIR}")
 
-    if multi_threaded:
-        cmd = f"cmake --build {BUILD_DIR} --target install -j"    
-        run(cmd, cwd=ROOT_DIR)    
-    else:
-        cmd = f"cmake --build {BUILD_DIR} --target install"
-        run(cmd, cwd=ROOT_DIR)    
+    cmd = f"cmake --build {BUILD_DIR} --target install"
+    if num_threads > 1:
+        cmd += f" -j {num_threads}"
+    run(cmd, cwd=ROOT_DIR)    
 
-if "__main__"==__name__:
 
-    build_local(multi_threaded=True)
+if "__main__" == __name__:
+    num_threads = int(os.getenv("BUILD_THREADS", "4"))
+    build_local(num_threads=num_threads)
diff --git a/cmake/extlib_json.cmake b/cmake/extlib_json.cmake
@@ -6,13 +6,13 @@ include(ExternalProject)
 include(CMakeParseArguments)
 
 set(JSON_URL https://github.com/nlohmann/json.git)
-#set(JSON_TAG v3.10.5)
+set(JSON_TAG v3.11.3)
 
 ExternalProject_Add(extlib_json
     PREFIX extlib_json
 
     GIT_REPOSITORY ${JSON_URL}
-    #GIT_TAG ${JSON_TAG}
+    GIT_TAG ${JSON_TAG}
 
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""

diff --git a/cmake/extlib_sentencepiece.cmake b/cmake/extlib_sentencepiece.cmake
@@ -39,7 +39,8 @@ ExternalProject_Add(extlib_sentencepiece
 
     BUILD_IN_SOURCE ON
     LOG_DOWNLOAD ON
-    LOG_BUILD ON)
+    # LOG_BUILD ON
+)
 
 add_library(sentencepiece STATIC IMPORTED)
 set_target_properties(sentencepiece PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libsentencepiece.a)